From b8ec453b810cbb8d42852f1882ae0d5ca1f596c9 Mon Sep 17 00:00:00 2001 From: seabbs <contact@samabbott.co.uk> Date: Thu, 31 Oct 2024 02:02:26 +0000 Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20@=20epiforec?= =?UTF-8?q?asts/scoringutils@bbdc711078467e6d8c877d3237d96da9c1470ed0=20?= =?UTF-8?q?=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dev/articles/scoring-rules.html | 2 +- dev/index.html | 2 +- dev/pkgdown.yml | 2 +- dev/reference/apply_metrics.html | 2 +- dev/reference/as_forecast_binary.html | 120 ++++++++++++++- dev/reference/as_forecast_doc_template.html | 144 ++++++++++++++++++ dev/reference/as_forecast_generic.html | 4 +- dev/reference/as_forecast_nominal.html | 139 +++++++++++++++-- dev/reference/as_forecast_point.html | 30 ++-- dev/reference/as_forecast_quantile.html | 143 ++++++++++++++--- dev/reference/as_forecast_sample.html | 77 +++++++++- dev/reference/assert_forecast.html | 59 ++----- dev/reference/assert_forecast_type.html | 2 +- dev/reference/check_duplicates.html | 4 +- dev/reference/clean_forecast.html | 2 +- dev/reference/example_binary.html | 4 +- dev/reference/example_nominal.html | 4 +- dev/reference/example_point.html | 4 +- dev/reference/example_quantile.html | 4 +- dev/reference/example_sample_continuous.html | 4 +- dev/reference/example_sample_discrete.html | 4 +- dev/reference/figures/metrics-nominal.png | Bin 0 -> 82676 bytes dev/reference/forecast_types.html | 48 +----- dev/reference/get_coverage.html | 8 +- dev/reference/get_duplicate_forecasts.html | 4 +- dev/reference/get_forecast_counts.html | 2 +- dev/reference/get_forecast_type.html | 2 +- dev/reference/get_forecast_unit.html | 10 +- .../get_metrics.forecast_binary.html | 10 +- .../get_metrics.forecast_nominal.html | 4 +- dev/reference/get_metrics.forecast_point.html | 4 +- .../get_metrics.forecast_quantile.html | 4 +- .../get_metrics.forecast_sample.html | 20 +-- dev/reference/get_metrics.html | 17 +-- dev/reference/get_pit_histogram.html | 2 +- dev/reference/get_protected_columns.html | 4 +- .../illustration-input-metric-nominal.html | 71 +++++++++ dev/reference/index.html | 14 +- dev/reference/is_forecast.html | 19 +-- dev/reference/new_forecast.html | 4 +- dev/reference/print.forecast.html | 3 +- dev/reference/quantile_to_interval.html | 2 +- dev/reference/sample_to_interval_long.html | 4 +- dev/reference/score.html | 91 +++-------- dev/reference/scoring-functions-nominal.html | 5 + dev/reference/select_metrics.html | 4 +- dev/reference/set_forecast_unit.html | 16 +- dev/reference/transform_forecasts.html | 2 +- dev/search.json | 2 +- dev/sitemap.xml | 3 +- 50 files changed, 803 insertions(+), 332 deletions(-) create mode 100644 dev/reference/as_forecast_doc_template.html create mode 100644 dev/reference/figures/metrics-nominal.png create mode 100644 dev/reference/illustration-input-metric-nominal.html diff --git a/dev/articles/scoring-rules.html b/dev/articles/scoring-rules.html index 65f28097b..ecd44eb9f 100644 --- a/dev/articles/scoring-rules.html +++ b/dev/articles/scoring-rules.html @@ -63,7 +63,7 @@ <h1>Scoring rules in `scoringutils`</h1> <h4 data-toc-skip class="author">Nikos Bosse</h4> - <h4 data-toc-skip class="date">2024-10-30</h4> + <h4 data-toc-skip class="date">2024-10-31</h4> <small class="dont-index">Source: <a href="https://github.com/epiforecasts/scoringutils/blob/main/vignettes/scoring-rules.Rmd" class="external-link"><code>vignettes/scoring-rules.Rmd</code></a></small> <div class="d-none name"><code>scoring-rules.Rmd</code></div> diff --git a/dev/index.html b/dev/index.html index 1072e933e..7ac6642d5 100644 --- a/dev/index.html +++ b/dev/index.html @@ -91,7 +91,7 @@ <h3 id="forecast-types">Forecast types<a class="anchor" aria-label="anchor" href <h3 id="input-formats-and-input-validation">Input formats and input validation<a class="anchor" aria-label="anchor" href="#input-formats-and-input-validation"></a> </h3> <p>The expected input format is generally a <code>data.frame</code> (or similar) with required columns <code>observed</code>, <code>predicted</code>, and <code>model</code> that holds the forecasts and observed values. Exact requirements depend on the forecast type. For more information, have a look at the <a href="https://drive.google.com/file/d/1URaMsXmHJ1twpLpMl1sl2HW4lPuUycoj/view?usp=drive_link" class="external-link">paper</a>, call <code>?as_forecast()</code>, or have a look at the example data provided in the package (<code>example_binary</code>, <code>example_point</code>, <code>example_quantile</code>, <code>example_sample_continuous</code>, <code>example_sample_discrete</code>).</p> -<p>Before scoring, input data needs to be validated and transformed into a forecast object using the function <code><a href="reference/as_forecast.html">as_forecast()</a></code>.</p> +<p>Before scoring, input data needs to be validated and transformed into a forecast object using the function <code>as_forecast()</code>.</p> <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r"> <code class="sourceCode R"><span><span class="va">forecast_quantile</span> <span class="op"><-</span> <span class="va">example_quantile</span> <span class="op">|></span></span> <span> <span class="fu"><a href="reference/as_forecast_quantile.html">as_forecast_quantile</a></span><span class="op">(</span></span> diff --git a/dev/pkgdown.yml b/dev/pkgdown.yml index e048f8a27..42dd57ab9 100644 --- a/dev/pkgdown.yml +++ b/dev/pkgdown.yml @@ -5,7 +5,7 @@ articles: Deprecated-functions: Deprecated-functions.html Deprecated-visualisations: Deprecated-visualisations.html scoring-rules: scoring-rules.html -last_built: 2024-10-30T21:32Z +last_built: 2024-10-31T02:01Z urls: reference: https://epiforecasts.io/scoringutils/reference article: https://epiforecasts.io/scoringutils/articles diff --git a/dev/reference/apply_metrics.html b/dev/reference/apply_metrics.html index 002b7407a..21a8f9901 100644 --- a/dev/reference/apply_metrics.html +++ b/dev/reference/apply_metrics.html @@ -69,7 +69,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-metrics">metrics<a class="anchor" aria-label="anchor" href="#arg-metrics"></a></dt> diff --git a/dev/reference/as_forecast_binary.html b/dev/reference/as_forecast_binary.html index c3d3be7bf..67a60f43d 100644 --- a/dev/reference/as_forecast_binary.html +++ b/dev/reference/as_forecast_binary.html @@ -1,7 +1,21 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for binary forecasts — as_forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for binary forecasts — as_forecast_binary"><meta name="description" content="Create a forecast object for binary forecasts. See more information on -forecast types and expected input formats by calling ?as_forecast()."><meta property="og:description" content="Create a forecast object for binary forecasts. See more information on -forecast types and expected input formats by calling ?as_forecast()."><meta name="robots" content="noindex"></head><body> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for binary forecasts — as_forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for binary forecasts — as_forecast_binary"><meta name="description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta property="og:description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -42,8 +56,15 @@ <h1>Create a <code>forecast</code> object for binary forecasts</h1> </div> <div class="ref-description section level2"> - <p>Create a <code>forecast</code> object for binary forecasts. See more information on -forecast types and expected input formats by calling <code>?</code><code><a href="as_forecast.html">as_forecast()</a></code>.</p> + <p>Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a <code>forecast</code> object. A forecast object is a <code>data.table</code> with +a class <code>forecast</code> and an additional class that depends on the forecast type.</p> +<p>The arguments <code>observed</code>, <code>predicted</code>, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument <code>forecast_unit</code>, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).</p> </div> <div class="section level2"> @@ -62,8 +83,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> @@ -85,16 +106,99 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen predicted values. This column will be renamed to "predicted".</p></dd> </dl></div> + <div class="section level2"> + <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> + <p>A <code>forecast</code> object of class <code>forecast_binary</code></p> + </div> + <div class="section level2"> + <h2 id="required-input">Required input<a class="anchor" aria-label="anchor" href="#required-input"></a></h2> + <p>The input needs to be a data.frame or similar with the following columns:</p><ul><li><p><code>observed</code>: <code>factor</code> with exactly two levels representing the observed +values. The highest factor level is assumed to be the reference level. +This means that corresponding value in <code>predicted</code> represent the +probability that the observed value is equal to the highest factor level.</p></li> +<li><p><code>predicted</code>: <code>numeric</code> with predicted probabilities, representing +the probability that the corresponding value in <code>observed</code> is equal to +the highest available factor level.</p></li> +</ul><p>For convenience, we recommend an additional column <code>model</code> holding the name +of the forecaster or model that produced a prediction, but this is not +strictly necessary.</p> +<p>See the <a href="example_binary.html">example_binary</a> data set for an example.</p> + </div> + <div class="section level2"> + <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> + <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows +of the data belong together and jointly form a single forecasts. This is +easy e.g. for point forecast, where there is one row per forecast. For +quantile or sample-based forecasts, however, there are multiple rows that +belong to a single forecast.</p> +<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the +combination of columns that uniquely identify a single forecast. +For example, we could have forecasts made by different models in various +locations at different time points, each for several weeks into the future. +The forecast unit could then be described as +<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. +<code>scoringutils</code> automatically tries to determine the unit of a single +forecast. It uses all existing columns for this, which means that no columns +must be present that are unrelated to the forecast unit. As a very simplistic +example, if you had an additional row, "even", that is one if the row number +is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> +then thinks that this column was relevant in defining the forecast unit.</p> +<p>In order to avoid issues, we recommend setting the forecast unit explicitly, +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> + </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other functions to create forecast objects: -<code><a href="as_forecast.html">as_forecast</a></code>, <code><a href="as_forecast_nominal.html">as_forecast_nominal</a>()</code>, <code><a href="as_forecast_point.html">as_forecast_point</a>()</code>, <code><a href="as_forecast_quantile.html">as_forecast_quantile</a>()</code>, <code><a href="as_forecast_sample.html">as_forecast_sample</a>()</code></p></div> </div> + <div class="section level2"> + <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-examples"></a></h2> + <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="fu">as_forecast_binary</span><span class="op">(</span></span></span> +<span class="r-in"><span> <span class="va">example_binary</span>,</span></span> +<span class="r-in"><span> predicted <span class="op">=</span> <span class="st">"predicted"</span>,</span></span> +<span class="r-in"><span> forecast_unit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"model"</span>, <span class="st">"target_type"</span>, <span class="st">"target_end_date"</span>,</span></span> +<span class="r-in"><span> <span class="st">"horizon"</span>, <span class="st">"location"</span><span class="op">)</span></span></span> +<span class="r-in"><span><span class="op">)</span></span></span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #00BBBB;">ℹ</span> Some rows containing NA values may be removed. This is fine if not</span> +<span class="r-msg co"><span class="r-pr">#></span> unexpected.</span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast type: </span>binary</span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast unit:</span></span> +<span class="r-msg co"><span class="r-pr">#></span> model, target_type, target_end_date, horizon, and location</span> +<span class="r-out co"><span class="r-pr">#></span> </span> +<span class="r-out co"><span class="r-pr">#></span> predicted observed model target_type target_end_date</span> +<span class="r-out co"><span class="r-pr">#></span> <num> <fctr> <char> <char> <Date></span> +<span class="r-out co"><span class="r-pr">#></span> 1: NA <NA> <NA> Cases 2021-01-02</span> +<span class="r-out co"><span class="r-pr">#></span> 2: NA <NA> <NA> Deaths 2021-01-02</span> +<span class="r-out co"><span class="r-pr">#></span> 3: NA <NA> <NA> Cases 2021-01-09</span> +<span class="r-out co"><span class="r-pr">#></span> 4: NA <NA> <NA> Deaths 2021-01-09</span> +<span class="r-out co"><span class="r-pr">#></span> 5: NA <NA> <NA> Cases 2021-01-16</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 1027: 0.250 0 EuroCOVIDhub-baseline Deaths 2021-07-24</span> +<span class="r-out co"><span class="r-pr">#></span> 1028: 0.475 0 UMass-MechBayes Deaths 2021-07-24</span> +<span class="r-out co"><span class="r-pr">#></span> 1029: 0.450 0 UMass-MechBayes Deaths 2021-07-24</span> +<span class="r-out co"><span class="r-pr">#></span> 1030: 0.375 0 epiforecasts-EpiNow2 Deaths 2021-07-24</span> +<span class="r-out co"><span class="r-pr">#></span> 1031: 0.300 0 epiforecasts-EpiNow2 Deaths 2021-07-24</span> +<span class="r-out co"><span class="r-pr">#></span> horizon location</span> +<span class="r-out co"><span class="r-pr">#></span> <num> <char></span> +<span class="r-out co"><span class="r-pr">#></span> 1: NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 2: NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 3: NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 4: NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 5: NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 1027: 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 1028: 3 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 1029: 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 1030: 3 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 1031: 2 IT</span> +</code></pre></div> + </div> </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> </nav></aside></div> diff --git a/dev/reference/as_forecast_doc_template.html b/dev/reference/as_forecast_doc_template.html new file mode 100644 index 000000000..c46795ea0 --- /dev/null +++ b/dev/reference/as_forecast_doc_template.html @@ -0,0 +1,144 @@ +<!DOCTYPE html> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>General information on creating a forecast object — as_forecast_doc_template • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="General information on creating a forecast object — as_forecast_doc_template"><meta name="description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta property="og:description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta name="robots" content="noindex"></head><body> + <a href="#main" class="visually-hidden-focusable">Skip to contents</a> + + + <nav class="navbar navbar-expand-lg fixed-top bg-light" data-bs-theme="light" aria-label="Site navigation"><div class="container"> + + <a class="navbar-brand me-2" href="../index.html">scoringutils</a> + + <small class="nav-text text-danger me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="In-development version">1.2.2.9000</small> + + + <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation"> + <span class="navbar-toggler-icon"></span> + </button> + + <div id="navbar" class="collapse navbar-collapse ms-3"> + <ul class="navbar-nav me-auto"><li class="active nav-item"><a class="nav-link" href="../reference/index.html">Reference</a></li> +<li class="nav-item"><a class="external-link nav-link" href="https://raw.githubusercontent.com/epiforecasts/scoringutils/main/inst/manuscript/manuscript.pdf">scoringutils paper</a></li> +<li class="nav-item dropdown"> + <button class="nav-link dropdown-toggle" type="button" id="dropdown-articles" data-bs-toggle="dropdown" aria-expanded="false" aria-haspopup="true">Articles</button> + <ul class="dropdown-menu" aria-labelledby="dropdown-articles"><li><a class="dropdown-item" href="../articles/Deprecated-functions.html">Deprecated functions</a></li> + <li><a class="dropdown-item" href="../articles/Deprecated-visualisations.html">Deprecated Visualisations</a></li> + <li><a class="dropdown-item" href="../articles/scoring-rules.html">Scoring rules in `scoringutils`</a></li> + </ul></li> + </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search"> + <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li> +<li class="nav-item"><a class="external-link nav-link" href="https://github.com/epiforecasts/scoringutils/" aria-label="GitHub"><span class="fa fab fa-github fa-lg"></span></a></li> + </ul></div> + + + </div> +</nav><div class="container template-reference-topic"> +<div class="row"> + <main id="main" class="col-md-9"><div class="page-header"> + + <h1>General information on creating a <code>forecast</code> object</h1> + <small class="dont-index">Source: <a href="https://github.com/epiforecasts/scoringutils/blob/main/R/documentation-templates.R" class="external-link"><code>R/documentation-templates.R</code></a></small> + <div class="d-none name"><code>as_forecast_doc_template.Rd</code></div> + </div> + + <div class="ref-description section level2"> + <p>Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a <code>forecast</code> object. A forecast object is a <code>data.table</code> with +a class <code>forecast</code> and an additional class that depends on the forecast type.</p> +<p>The arguments <code>observed</code>, <code>predicted</code>, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument <code>forecast_unit</code>, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).</p> + </div> + + + <div class="section level2"> + <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#arguments"></a></h2> + + +<dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> +<dd><p>A data.frame (or similar) with predicted and observed values. +See the details section of for additional information +on the required input format.</p></dd> + + +<dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> +<dd><p>(optional) Name of the columns in <code>data</code> (after +any renaming of columns) that denote the unit of a +single forecast. See <code><a href="get_forecast_unit.html">get_forecast_unit()</a></code> for details. +If <code>NULL</code> (the default), all columns that are not required columns are +assumed to form the unit of a single forecast. If specified, all columns +that are not part of the forecast unit (or required columns) will be removed.</p></dd> + + +<dt id="arg-observed">observed<a class="anchor" aria-label="anchor" href="#arg-observed"></a></dt> +<dd><p>(optional) Name of the column in <code>data</code> that contains the +observed values. This column will be renamed to "observed".</p></dd> + + +<dt id="arg-predicted">predicted<a class="anchor" aria-label="anchor" href="#arg-predicted"></a></dt> +<dd><p>(optional) Name of the column in <code>data</code> that contains the +predicted values. This column will be renamed to "predicted".</p></dd> + +</dl></div> + <div class="section level2"> + <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> + <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows +of the data belong together and jointly form a single forecasts. This is +easy e.g. for point forecast, where there is one row per forecast. For +quantile or sample-based forecasts, however, there are multiple rows that +belong to a single forecast.</p> +<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the +combination of columns that uniquely identify a single forecast. +For example, we could have forecasts made by different models in various +locations at different time points, each for several weeks into the future. +The forecast unit could then be described as +<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. +<code>scoringutils</code> automatically tries to determine the unit of a single +forecast. It uses all existing columns for this, which means that no columns +must be present that are unrelated to the forecast unit. As a very simplistic +example, if you had an additional row, "even", that is one if the row number +is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> +then thinks that this column was relevant in defining the forecast unit.</p> +<p>In order to avoid issues, we recommend setting the forecast unit explicitly, +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> + </div> + + </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> + </nav></aside></div> + + + <footer><div class="pkgdown-footer-left"> + <p>Developed by <a href="https://followtheargument.org/" class="external-link">Nikos Bosse</a>, <a href="https://www.samabbott.co.uk/" class="external-link">Sam Abbott</a>, Hugo Gruson, Sebastian Funk.</p> +</div> + +<div class="pkgdown-footer-right"> + <p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.1.1.9000.</p> +</div> + + </footer></div> + + + + + + </body></html> + diff --git a/dev/reference/as_forecast_generic.html b/dev/reference/as_forecast_generic.html index dd9d3adeb..f4382e192 100644 --- a/dev/reference/as_forecast_generic.html +++ b/dev/reference/as_forecast_generic.html @@ -59,8 +59,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> diff --git a/dev/reference/as_forecast_nominal.html b/dev/reference/as_forecast_nominal.html index d528766cf..d3ec9615d 100644 --- a/dev/reference/as_forecast_nominal.html +++ b/dev/reference/as_forecast_nominal.html @@ -1,9 +1,21 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for nominal forecasts — as_forecast_nominal • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for nominal forecasts — as_forecast_nominal"><meta name="description" content="Nominal forecasts are a form of categorical forecasts where the possible -outcomes that the observed values can assume are not ordered. In that sense, -Nominal forecasts represent a generalisation of binary forecasts."><meta property="og:description" content="Nominal forecasts are a form of categorical forecasts where the possible -outcomes that the observed values can assume are not ordered. In that sense, -Nominal forecasts represent a generalisation of binary forecasts."><meta name="robots" content="noindex"></head><body> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for nominal forecasts — as_forecast_nominal • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for nominal forecasts — as_forecast_nominal"><meta name="description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta property="og:description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -44,9 +56,15 @@ <h1>Create a <code>forecast</code> object for nominal forecasts</h1> </div> <div class="ref-description section level2"> - <p>Nominal forecasts are a form of categorical forecasts where the possible -outcomes that the observed values can assume are not ordered. In that sense, -Nominal forecasts represent a generalisation of binary forecasts.</p> + <p>Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a <code>forecast</code> object. A forecast object is a <code>data.table</code> with +a class <code>forecast</code> and an additional class that depends on the forecast type.</p> +<p>The arguments <code>observed</code>, <code>predicted</code>, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument <code>forecast_unit</code>, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).</p> </div> <div class="section level2"> @@ -66,8 +84,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> @@ -92,20 +110,115 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dt id="arg-predicted-label">predicted_label<a class="anchor" aria-label="anchor" href="#arg-predicted-label"></a></dt> <dd><p>(optional) Name of the column in <code>data</code> that denotes the outcome to which a predicted probability corresponds to. -This column will be renamed to "predicted_label". Only applicable to -nominal forecasts.</p></dd> +This column will be renamed to "predicted_label".</p></dd> </dl></div> + <div class="section level2"> + <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> + <p>A <code>forecast</code> object of class <code>forecast_nominal</code></p> + </div> + <div class="section level2"> + <h2 id="details">Details<a class="anchor" aria-label="anchor" href="#details"></a></h2> + <p>Nominal forecasts are a form of categorical forecasts and represent a +generalisation of binary forecasts to multiple outcomes. The possible +outcomes that the observed values can assume are not ordered.</p> + </div> + <div class="section level2"> + <h2 id="required-input">Required input<a class="anchor" aria-label="anchor" href="#required-input"></a></h2> + <p>The input needs to be a data.frame or similar with the following columns:</p><ul><li><p><code>observed</code>: Column with observed values of type <code>factor</code> with N levels, +where N is the number of possible outcomes. +The levels of the factor represent the possible outcomes that +the observed values can assume.</p></li> +<li><p><code>predicted</code>: <code>numeric</code> column with predicted probabilities. The values +represent the probability that the observed value is equal to the factor +level denoted in <code>predicted_label</code>. Note that forecasts must be complete, +i.e. there must be a probability assigned to every possible outcome and +those probabilities must sum to one.</p></li> +<li><p><code>predicted_label</code>: <code>factor</code> with N levels, denoting the outcome that the +probabilities in <code>predicted</code> correspond to.</p></li> +</ul><p>For convenience, we recommend an additional column <code>model</code> holding the name +of the forecaster or model that produced a prediction, but this is not +strictly necessary.</p> +<p>See the <a href="example_nominal.html">example_nominal</a> data set for an example.</p> + </div> + <div class="section level2"> + <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> + <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows +of the data belong together and jointly form a single forecasts. This is +easy e.g. for point forecast, where there is one row per forecast. For +quantile or sample-based forecasts, however, there are multiple rows that +belong to a single forecast.</p> +<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the +combination of columns that uniquely identify a single forecast. +For example, we could have forecasts made by different models in various +locations at different time points, each for several weeks into the future. +The forecast unit could then be described as +<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. +<code>scoringutils</code> automatically tries to determine the unit of a single +forecast. It uses all existing columns for this, which means that no columns +must be present that are unrelated to the forecast unit. As a very simplistic +example, if you had an additional row, "even", that is one if the row number +is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> +then thinks that this column was relevant in defining the forecast unit.</p> +<p>In order to avoid issues, we recommend setting the forecast unit explicitly, +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> + </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other functions to create forecast objects: -<code><a href="as_forecast.html">as_forecast</a></code>, <code><a href="as_forecast_binary.html">as_forecast_binary</a>()</code>, <code><a href="as_forecast_point.html">as_forecast_point</a>()</code>, <code><a href="as_forecast_quantile.html">as_forecast_quantile</a>()</code>, <code><a href="as_forecast_sample.html">as_forecast_sample</a>()</code></p></div> </div> + <div class="section level2"> + <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-examples"></a></h2> + <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="fu">as_forecast_nominal</span><span class="op">(</span></span></span> +<span class="r-in"><span> <span class="fu"><a href="https://rdrr.io/r/stats/na.fail.html" class="external-link">na.omit</a></span><span class="op">(</span><span class="va">example_nominal</span><span class="op">)</span>,</span></span> +<span class="r-in"><span> predicted <span class="op">=</span> <span class="st">"predicted"</span>,</span></span> +<span class="r-in"><span> forecast_unit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"model"</span>, <span class="st">"target_type"</span>, <span class="st">"target_end_date"</span>,</span></span> +<span class="r-in"><span> <span class="st">"horizon"</span>, <span class="st">"location"</span><span class="op">)</span></span></span> +<span class="r-in"><span><span class="op">)</span></span></span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast type: </span>nominal</span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast unit:</span></span> +<span class="r-msg co"><span class="r-pr">#></span> model, target_type, target_end_date, horizon, and location</span> +<span class="r-out co"><span class="r-pr">#></span> </span> +<span class="r-wrn co"><span class="r-pr">#></span> <span class="warning">Warning: </span><span style="color: #BBBB00;">!</span> Error in validating forecast object: Error in assert_forecast(forecast = out,</span> +<span class="r-wrn co"><span class="r-pr">#></span> verbose = FALSE) : <span style="color: #BBBB00;">!</span> Found incomplete forecasts <span style="color: #00BBBB;">ℹ</span> For a nominal forecast, all</span> +<span class="r-wrn co"><span class="r-pr">#></span> possible outcomes must be assigned a probability explicitly. <span style="color: #00BBBB;">ℹ</span> Found first</span> +<span class="r-wrn co"><span class="r-pr">#></span> missing probabilities in the forecast identified by <span style="font-style: italic;">model == NA</span>, <span style="font-style: italic;">target_type</span></span> +<span class="r-wrn co"><span class="r-pr">#></span> <span style="font-style: italic;">== NA</span>, <span style="font-style: italic;">target_end_date == NA</span>, <span style="font-style: italic;">horizon == NA</span>, and <span style="font-style: italic;">location == NA</span></span> +<span class="r-out co"><span class="r-pr">#></span> observed predicted_label predicted model target_type</span> +<span class="r-out co"><span class="r-pr">#></span> <fctr> <fctr> <num> <char> <char></span> +<span class="r-out co"><span class="r-pr">#></span> 1: low low 0.525 EuroCOVIDhub-ensemble Cases</span> +<span class="r-out co"><span class="r-pr">#></span> 2: low low 0.075 EuroCOVIDhub-baseline Cases</span> +<span class="r-out co"><span class="r-pr">#></span> 3: low low 0.150 epiforecasts-EpiNow2 Cases</span> +<span class="r-out co"><span class="r-pr">#></span> 4: medium low 0.100 EuroCOVIDhub-ensemble Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 5: medium low 0.275 EuroCOVIDhub-baseline Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 2657: low medium 0.300 EuroCOVIDhub-baseline Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 2658: medium medium 0.850 UMass-MechBayes Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 2659: low medium 0.825 UMass-MechBayes Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 2660: medium medium 0.275 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 2661: low medium 0.375 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> target_end_date horizon location</span> +<span class="r-out co"><span class="r-pr">#></span> <Date> <num> <char></span> +<span class="r-out co"><span class="r-pr">#></span> 1: 2021-05-08 1 DE</span> +<span class="r-out co"><span class="r-pr">#></span> 2: 2021-05-08 1 DE</span> +<span class="r-out co"><span class="r-pr">#></span> 3: 2021-05-08 1 DE</span> +<span class="r-out co"><span class="r-pr">#></span> 4: 2021-05-08 1 DE</span> +<span class="r-out co"><span class="r-pr">#></span> 5: 2021-05-08 1 DE</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 2657: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 2658: 2021-07-24 3 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 2659: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 2660: 2021-07-24 3 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 2661: 2021-07-24 2 IT</span> +</code></pre></div> + </div> </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> </nav></aside></div> diff --git a/dev/reference/as_forecast_point.html b/dev/reference/as_forecast_point.html index 4ffaeca3c..42dd5490d 100644 --- a/dev/reference/as_forecast_point.html +++ b/dev/reference/as_forecast_point.html @@ -1,10 +1,6 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for point forecasts — as_forecast_point • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for point forecasts — as_forecast_point"><meta name="description" content="Create a forecast object for point forecasts. See more information on -forecast types and expected input formats by calling ?as_forecast(). -When converting a forecast_quantile object into a forecast_point object, -the 0.5 quantile is extracted and returned as the point forecast."><meta property="og:description" content="Create a forecast object for point forecasts. See more information on -forecast types and expected input formats by calling ?as_forecast(). -When converting a forecast_quantile object into a forecast_point object, +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for point forecasts — as_forecast_point • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for point forecasts — as_forecast_point"><meta name="description" content="When converting a forecast_quantile object into a forecast_point object, +the 0.5 quantile is extracted and returned as the point forecast."><meta property="og:description" content="When converting a forecast_quantile object into a forecast_point object, the 0.5 quantile is extracted and returned as the point forecast."><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -46,9 +42,7 @@ <h1>Create a <code>forecast</code> object for point forecasts</h1> </div> <div class="ref-description section level2"> - <p>Create a <code>forecast</code> object for point forecasts. See more information on -forecast types and expected input formats by calling <code>?</code><code><a href="as_forecast.html">as_forecast()</a></code>.</p> -<p>When converting a <code>forecast_quantile</code> object into a <code>forecast_point</code> object, + <p>When converting a <code>forecast_quantile</code> object into a <code>forecast_point</code> object, the 0.5 quantile is extracted and returned as the point forecast.</p> </div> @@ -75,8 +69,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg--">...<a class="anchor" aria-label="anchor" href="#arg--"></a></dt> @@ -102,10 +96,22 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen predicted values. This column will be renamed to "predicted".</p></dd> </dl></div> + <div class="section level2"> + <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> + <p>A <code>forecast</code> object of class <code>forecast_point</code></p> + </div> + <div class="section level2"> + <h2 id="required-input">Required input<a class="anchor" aria-label="anchor" href="#required-input"></a></h2> + <p>The input needs to be a data.frame or similar with the following columns:</p><ul><li><p><code>observed</code>: Column of type <code>numeric</code> with observed values.</p></li> +<li><p><code>predicted</code>: Column of type <code>numeric</code> with predicted values.</p></li> +</ul><p>For convenience, we recommend an additional column <code>model</code> holding the name +of the forecaster or model that produced a prediction, but this is not +strictly necessary.</p> +<p>See the <a href="example_point.html">example_point</a> data set for an example.</p> + </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other functions to create forecast objects: -<code><a href="as_forecast.html">as_forecast</a></code>, <code><a href="as_forecast_binary.html">as_forecast_binary</a>()</code>, <code><a href="as_forecast_nominal.html">as_forecast_nominal</a>()</code>, <code><a href="as_forecast_quantile.html">as_forecast_quantile</a>()</code>, diff --git a/dev/reference/as_forecast_quantile.html b/dev/reference/as_forecast_quantile.html index d7be578de..4c8b97bd9 100644 --- a/dev/reference/as_forecast_quantile.html +++ b/dev/reference/as_forecast_quantile.html @@ -1,17 +1,21 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for quantile-based forecasts — as_forecast_quantile • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for quantile-based forecasts — as_forecast_quantile"><meta name="description" content="Create a forecast object for quantile-based forecasts. See more information -on forecast types and expected input formats by calling ?as_forecast(). -When creating a forecast_quantile object from a forecast_sample object, -the quantiles are estimated by computing empircal quantiles from the samples -via quantile(). Note that empirical quantiles are a biased estimator for -the true quantiles in particular in the tails of the distribution and -when the number of available samples is low."><meta property="og:description" content="Create a forecast object for quantile-based forecasts. See more information -on forecast types and expected input formats by calling ?as_forecast(). -When creating a forecast_quantile object from a forecast_sample object, -the quantiles are estimated by computing empircal quantiles from the samples -via quantile(). Note that empirical quantiles are a biased estimator for -the true quantiles in particular in the tails of the distribution and -when the number of available samples is low."><meta name="robots" content="noindex"></head><body> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for quantile-based forecasts — as_forecast_quantile • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for quantile-based forecasts — as_forecast_quantile"><meta name="description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta property="og:description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -52,13 +56,15 @@ <h1>Create a <code>forecast</code> object for quantile-based forecasts</h1> </div> <div class="ref-description section level2"> - <p>Create a <code>forecast</code> object for quantile-based forecasts. See more information -on forecast types and expected input formats by calling <code>?</code><code><a href="as_forecast.html">as_forecast()</a></code>.</p> -<p>When creating a <code>forecast_quantile</code> object from a <code>forecast_sample</code> object, -the quantiles are estimated by computing empircal quantiles from the samples -via <code><a href="https://rdrr.io/r/stats/quantile.html" class="external-link">quantile()</a></code>. Note that empirical quantiles are a biased estimator for -the true quantiles in particular in the tails of the distribution and -when the number of available samples is low.</p> + <p>Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a <code>forecast</code> object. A forecast object is a <code>data.table</code> with +a class <code>forecast</code> and an additional class that depends on the forecast type.</p> +<p>The arguments <code>observed</code>, <code>predicted</code>, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument <code>forecast_unit</code>, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).</p> </div> <div class="section level2"> @@ -90,8 +96,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg--">...<a class="anchor" aria-label="anchor" href="#arg--"></a></dt> @@ -134,16 +140,107 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen information, see <code><a href="https://rdrr.io/r/stats/quantile.html" class="external-link">quantile()</a></code>.</p></dd> </dl></div> + <div class="section level2"> + <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> + <p>A <code>forecast</code> object of class <code>forecast_quantile</code></p> + </div> + <div class="section level2"> + <h2 id="required-input">Required input<a class="anchor" aria-label="anchor" href="#required-input"></a></h2> + <p>The input needs to be a data.frame or similar with the following columns:</p><ul><li><p><code>observed</code>: Column of type <code>numeric</code> with observed values.</p></li> +<li><p><code>predicted</code>: Column of type <code>numeric</code> with predicted values. Predicted +values represent quantiles of the predictive distribution.</p></li> +<li><p><code>quantile_level</code>: Column of type <code>numeric</code>, denoting the quantile level of +the corresponding predicted value. +Quantile levels must be between 0 and 1.</p></li> +</ul><p>For convenience, we recommend an additional column <code>model</code> holding the name +of the forecaster or model that produced a prediction, but this is not +strictly necessary.</p> +<p>See the <a href="example_quantile.html">example_quantile</a> data set for an example.</p> + </div> + <div class="section level2"> + <h2 id="converting-from-forecast-sample-to-forecast-quantile">Converting from <code>forecast_sample</code> to <code>forecast_quantile</code><a class="anchor" aria-label="anchor" href="#converting-from-forecast-sample-to-forecast-quantile"></a></h2> + <p>When creating a <code>forecast_quantile</code> object from a <code>forecast_sample</code> object, +the quantiles are estimated by computing empircal quantiles from the samples +via <code><a href="https://rdrr.io/r/stats/quantile.html" class="external-link">quantile()</a></code>. Note that empirical quantiles are a biased estimator for +the true quantiles in particular in the tails of the distribution and +when the number of available samples is low.</p> + </div> + <div class="section level2"> + <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> + <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows +of the data belong together and jointly form a single forecasts. This is +easy e.g. for point forecast, where there is one row per forecast. For +quantile or sample-based forecasts, however, there are multiple rows that +belong to a single forecast.</p> +<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the +combination of columns that uniquely identify a single forecast. +For example, we could have forecasts made by different models in various +locations at different time points, each for several weeks into the future. +The forecast unit could then be described as +<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. +<code>scoringutils</code> automatically tries to determine the unit of a single +forecast. It uses all existing columns for this, which means that no columns +must be present that are unrelated to the forecast unit. As a very simplistic +example, if you had an additional row, "even", that is one if the row number +is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> +then thinks that this column was relevant in defining the forecast unit.</p> +<p>In order to avoid issues, we recommend setting the forecast unit explicitly, +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> + </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other functions to create forecast objects: -<code><a href="as_forecast.html">as_forecast</a></code>, <code><a href="as_forecast_binary.html">as_forecast_binary</a>()</code>, <code><a href="as_forecast_nominal.html">as_forecast_nominal</a>()</code>, <code><a href="as_forecast_point.html">as_forecast_point</a>()</code>, <code><a href="as_forecast_sample.html">as_forecast_sample</a>()</code></p></div> </div> + <div class="section level2"> + <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-examples"></a></h2> + <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="fu">as_forecast_quantile</span><span class="op">(</span></span></span> +<span class="r-in"><span> <span class="va">example_quantile</span>,</span></span> +<span class="r-in"><span> predicted <span class="op">=</span> <span class="st">"predicted"</span>,</span></span> +<span class="r-in"><span> forecast_unit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"model"</span>, <span class="st">"target_type"</span>, <span class="st">"target_end_date"</span>,</span></span> +<span class="r-in"><span> <span class="st">"horizon"</span>, <span class="st">"location"</span><span class="op">)</span></span></span> +<span class="r-in"><span><span class="op">)</span></span></span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #00BBBB;">ℹ</span> Some rows containing NA values may be removed. This is fine if not</span> +<span class="r-msg co"><span class="r-pr">#></span> unexpected.</span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast type: </span>quantile</span> +<span class="r-msg co"><span class="r-pr">#></span> <span style="color: #0000BB;">Forecast unit:</span></span> +<span class="r-msg co"><span class="r-pr">#></span> model, target_type, target_end_date, horizon, and location</span> +<span class="r-out co"><span class="r-pr">#></span> </span> +<span class="r-out co"><span class="r-pr">#></span> Key: <location, target_end_date, target_type></span> +<span class="r-out co"><span class="r-pr">#></span> observed quantile_level predicted model target_type</span> +<span class="r-out co"><span class="r-pr">#></span> <num> <num> <int> <char> <char></span> +<span class="r-out co"><span class="r-pr">#></span> 1: 127300 NA NA <NA> Cases</span> +<span class="r-out co"><span class="r-pr">#></span> 2: 4534 NA NA <NA> Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 3: 154922 NA NA <NA> Cases</span> +<span class="r-out co"><span class="r-pr">#></span> 4: 6117 NA NA <NA> Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 5: 110183 NA NA <NA> Cases</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths</span> +<span class="r-out co"><span class="r-pr">#></span> target_end_date horizon location</span> +<span class="r-out co"><span class="r-pr">#></span> <Date> <num> <char></span> +<span class="r-out co"><span class="r-pr">#></span> 1: 2021-01-02 NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 2: 2021-01-02 NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 3: 2021-01-09 NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 4: 2021-01-09 NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> 5: 2021-01-16 NA DE</span> +<span class="r-out co"><span class="r-pr">#></span> --- </span> +<span class="r-out co"><span class="r-pr">#></span> 20541: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 20542: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 20543: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 20544: 2021-07-24 2 IT</span> +<span class="r-out co"><span class="r-pr">#></span> 20545: 2021-07-24 2 IT</span> +</code></pre></div> + </div> </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> </nav></aside></div> diff --git a/dev/reference/as_forecast_sample.html b/dev/reference/as_forecast_sample.html index 4deae786e..65ed25a76 100644 --- a/dev/reference/as_forecast_sample.html +++ b/dev/reference/as_forecast_sample.html @@ -1,5 +1,21 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for sample-based forecasts — as_forecast_sample • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for sample-based forecasts — as_forecast_sample"><meta name="description" content="Create a forecast object for sample-based forecasts"><meta property="og:description" content="Create a forecast object for sample-based forecasts"><meta name="robots" content="noindex"></head><body> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Create a forecast object for sample-based forecasts — as_forecast_sample • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Create a forecast object for sample-based forecasts — as_forecast_sample"><meta name="description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta property="og:description" content='Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a forecast object. A forecast object is a data.table with +a class forecast and an additional class that depends on the forecast type. +The arguments observed, predicted, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument forecast_unit, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).'><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -40,7 +56,15 @@ <h1>Create a <code>forecast</code> object for sample-based forecasts</h1> </div> <div class="ref-description section level2"> - <p>Create a <code>forecast</code> object for sample-based forecasts</p> + <p>Process and validate a data.frame (or similar) or similar with forecasts +and observations. If the input passes all input checks, those functions will +be converted to a <code>forecast</code> object. A forecast object is a <code>data.table</code> with +a class <code>forecast</code> and an additional class that depends on the forecast type.</p> +<p>The arguments <code>observed</code>, <code>predicted</code>, etc. make it possible to rename +existing columns of the input data to match the required columns for a +forecast object. Using the argument <code>forecast_unit</code>, you can specify +the columns that uniquely identify a single forecast (and thereby removing +other, unneeded columns. See section "Forecast Unit" below for details).</p> </div> <div class="section level2"> @@ -60,8 +84,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> @@ -85,14 +109,53 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dt id="arg-sample-id">sample_id<a class="anchor" aria-label="anchor" href="#arg-sample-id"></a></dt> <dd><p>(optional) Name of the column in <code>data</code> that contains the -sample id. This column will be renamed to "sample_id". Only applicable to -sample-based forecasts.</p></dd> +sample id. This column will be renamed to "sample_id".</p></dd> </dl></div> + <div class="section level2"> + <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> + <p>A <code>forecast</code> object of class <code>forecast_sample</code></p> + </div> + <div class="section level2"> + <h2 id="required-input">Required input<a class="anchor" aria-label="anchor" href="#required-input"></a></h2> + <p>The input needs to be a data.frame or similar with the following columns:</p><ul><li><p><code>observed</code>: Column of type <code>numeric</code> with observed values.</p></li> +<li><p><code>predicted</code>: Column of type <code>numeric</code> with predicted values. Predicted +values represent random samples from the predictive distribution.</p></li> +<li><p><code>sample_id</code>: Column of any type with unique identifiers +(unique within a single forecast) for each sample.</p></li> +</ul><p>For convenience, we recommend an additional column <code>model</code> holding the name +of the forecaster or model that produced a prediction, but this is not +strictly necessary.</p> +<p>See the <a href="example_sample_continuous.html">example_sample_continuous</a> and <a href="example_sample_discrete.html">example_sample_discrete</a> data set +for an example</p> + </div> + <div class="section level2"> + <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> + <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows +of the data belong together and jointly form a single forecasts. This is +easy e.g. for point forecast, where there is one row per forecast. For +quantile or sample-based forecasts, however, there are multiple rows that +belong to a single forecast.</p> +<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the +combination of columns that uniquely identify a single forecast. +For example, we could have forecasts made by different models in various +locations at different time points, each for several weeks into the future. +The forecast unit could then be described as +<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. +<code>scoringutils</code> automatically tries to determine the unit of a single +forecast. It uses all existing columns for this, which means that no columns +must be present that are unrelated to the forecast unit. As a very simplistic +example, if you had an additional row, "even", that is one if the row number +is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> +then thinks that this column was relevant in defining the forecast unit.</p> +<p>In order to avoid issues, we recommend setting the forecast unit explicitly, +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> + </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other functions to create forecast objects: -<code><a href="as_forecast.html">as_forecast</a></code>, <code><a href="as_forecast_binary.html">as_forecast_binary</a>()</code>, <code><a href="as_forecast_nominal.html">as_forecast_nominal</a>()</code>, <code><a href="as_forecast_point.html">as_forecast_point</a>()</code>, diff --git a/dev/reference/assert_forecast.html b/dev/reference/assert_forecast.html index c037e7d84..c56cdba8f 100644 --- a/dev/reference/assert_forecast.html +++ b/dev/reference/assert_forecast.html @@ -1,9 +1,13 @@ <!DOCTYPE html> <!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary"><meta name="description" content="Assert that an object is a forecast object (i.e. a data.table with a class -forecast and an additional class forecast_* corresponding to the forecast -type)."><meta property="og:description" content="Assert that an object is a forecast object (i.e. a data.table with a class -forecast and an additional class forecast_* corresponding to the forecast -type)."><meta name="robots" content="noindex"></head><body> +forecast and an additional class forecast_&lt;type&gt; corresponding to the +forecast type). +See the corresponding assert_forecast_&lt;type&gt; functions for more details on +the required input formats."><meta property="og:description" content="Assert that an object is a forecast object (i.e. a data.table with a class +forecast and an additional class forecast_&lt;type&gt; corresponding to the +forecast type). +See the corresponding assert_forecast_&lt;type&gt; functions for more details on +the required input formats."><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -45,8 +49,10 @@ <h1>Assert that input is a forecast object and passes validations</h1> <div class="ref-description section level2"> <p>Assert that an object is a forecast object (i.e. a <code>data.table</code> with a class -<code>forecast</code> and an additional class <code>forecast_*</code> corresponding to the forecast -type).</p> +<code>forecast</code> and an additional class <code>forecast_<type></code> corresponding to the +forecast type).</p> +<p>See the corresponding <code>assert_forecast_<type></code> functions for more details on +the required input formats.</p> </div> <div class="section level2"> @@ -75,7 +81,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-forecast-type">forecast_type<a class="anchor" aria-label="anchor" href="#arg-forecast-type"></a></dt> @@ -101,45 +107,6 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> <p>Returns <code>NULL</code> invisibly.</p> </div> - <div class="section level2"> - <h2 id="forecast-types-and-input-formats">Forecast types and input formats<a class="anchor" aria-label="anchor" href="#forecast-types-and-input-formats"></a></h2> - <p>Various different forecast types / forecast formats are supported. At the -moment, those are:</p><ul><li><p>point forecasts</p></li> -<li><p>binary forecasts ("soft binary classification")</p></li> -<li><p>nominal forecasts ("soft classification with multiple unordered classes")</p></li> -<li><p>Probabilistic forecasts in a quantile-based format (a forecast is -represented as a set of predictive quantiles)</p></li> -<li><p>Probabilistic forecasts in a sample-based format (a forecast is represented -as a set of predictive samples)</p></li> -</ul><p>Forecast types are determined based on the columns present in the input data. -Here is an overview of the required format for each forecast type: - - </p><div style="text-align: left"> - <img src="figures/required-inputs.png" style="width:750px;max-width:100%;"></div> -<p><em>All forecast types</em> require a data.frame or similar with columns <code>observed</code> -<code>predicted</code>, and <code>model</code>.</p> -<p><em>Point forecasts</em> require a column <code>observed</code> of type numeric and a column -<code>predicted</code> of type numeric.</p> -<p><em>Binary forecasts</em> require a column <code>observed</code> of type factor with exactly -two levels and a column <code>predicted</code> of type numeric with probabilities, -corresponding to the probability that <code>observed</code> is equal to the second -factor level. See details <a href="scoring-functions-binary.html">here</a> for more information.</p> -<p><em>Nominal forecasts</em> require a column <code>observed</code> of type factor with N levels, -(where N is the number of possible outcomes), a column <code>predicted</code> of type -numeric with probabilities (which sum to one across all possible outcomes), -and a column <code>predicted_label</code> of type factor with N levels, denoting the -outcome for which a probability is given. Forecasts must be complete, i.e. -there must be a probability assigned to every possible outcome.</p> -<p><em>Quantile-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>quantile_level</code> of type -numeric with quantile-levels (between 0 and 1).</p> -<p><em>Sample-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>sample_id</code> of type -numeric with sample indices.</p> -<p>For more information see the vignettes and the example data -(<a href="example_quantile.html">example_quantile</a>, <a href="example_sample_continuous.html">example_sample_continuous</a>, <a href="example_sample_discrete.html">example_sample_discrete</a>, -<code><a href="example_point.html">example_point()</a></code>, <a href="example_binary.html">example_binary</a>, and <a href="example_nominal.html">example_nominal</a>).</p> - </div> <div class="section level2"> <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-examples"></a></h2> diff --git a/dev/reference/assert_forecast_type.html b/dev/reference/assert_forecast_type.html index 72b1734eb..f2b8d578a 100644 --- a/dev/reference/assert_forecast_type.html +++ b/dev/reference/assert_forecast_type.html @@ -53,7 +53,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> -<dd><p>A forecast object (see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +<dd><p>A forecast object.</p></dd> <dt id="arg-actual">actual<a class="anchor" aria-label="anchor" href="#arg-actual"></a></dt> diff --git a/dev/reference/check_duplicates.html b/dev/reference/check_duplicates.html index f6a83b59e..e01530f85 100644 --- a/dev/reference/check_duplicates.html +++ b/dev/reference/check_duplicates.html @@ -57,8 +57,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> </dl></div> <div class="section level2"> diff --git a/dev/reference/clean_forecast.html b/dev/reference/clean_forecast.html index ae662f300..14b9f8b3f 100644 --- a/dev/reference/clean_forecast.html +++ b/dev/reference/clean_forecast.html @@ -57,7 +57,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-copy">copy<a class="anchor" aria-label="anchor" href="#arg-copy"></a></dt> diff --git a/dev/reference/example_binary.html b/dev/reference/example_binary.html index 9e0f47e8e..6420cf915 100644 --- a/dev/reference/example_binary.html +++ b/dev/reference/example_binary.html @@ -53,8 +53,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_binary</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_binary</code> (see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>) +with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>location_name</dt> diff --git a/dev/reference/example_nominal.html b/dev/reference/example_nominal.html index 684bcfa0f..a396473c0 100644 --- a/dev/reference/example_nominal.html +++ b/dev/reference/example_nominal.html @@ -53,8 +53,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_nominal</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_nominal</code> +(see <code><a href="as_forecast_nominal.html">as_forecast_nominal()</a></code>) with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>target_end_date</dt> diff --git a/dev/reference/example_point.html b/dev/reference/example_point.html index a915113c5..5ccee3130 100644 --- a/dev/reference/example_point.html +++ b/dev/reference/example_point.html @@ -56,8 +56,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_point</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_point</code> (see <code><a href="as_forecast_point.html">as_forecast_point()</a></code>) +with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>target_end_date</dt> diff --git a/dev/reference/example_quantile.html b/dev/reference/example_quantile.html index 29c4cffdd..30a3f9276 100644 --- a/dev/reference/example_quantile.html +++ b/dev/reference/example_quantile.html @@ -53,8 +53,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_quantile</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_quantile</code> +(see <code><a href="as_forecast_quantile.html">as_forecast_quantile()</a></code>) with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>target_end_date</dt> diff --git a/dev/reference/example_sample_continuous.html b/dev/reference/example_sample_continuous.html index b01834df7..5c4f8eebb 100644 --- a/dev/reference/example_sample_continuous.html +++ b/dev/reference/example_sample_continuous.html @@ -53,8 +53,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_sample</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_sample</code> (see <code><a href="as_forecast_sample.html">as_forecast_sample()</a></code>) +with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>target_end_date</dt> diff --git a/dev/reference/example_sample_discrete.html b/dev/reference/example_sample_discrete.html index 02ee70d49..c1213d05a 100644 --- a/dev/reference/example_sample_discrete.html +++ b/dev/reference/example_sample_discrete.html @@ -53,8 +53,8 @@ <h2 id="ref-usage">Usage<a class="anchor" aria-label="anchor" href="#ref-usage"> <div class="section level2"> <h2 id="format">Format<a class="anchor" aria-label="anchor" href="#format"></a></h2> - <p>An object of class <code>forecast_sample</code> (see <code><a href="as_forecast.html">as_forecast()</a></code>) with the -following columns:</p><dl><dt>location</dt> + <p>An object of class <code>forecast_sample</code> (see <code><a href="as_forecast_sample.html">as_forecast_sample()</a></code>) +with the following columns:</p><dl><dt>location</dt> <dd><p>the country for which a prediction was made</p></dd> <dt>target_end_date</dt> diff --git a/dev/reference/figures/metrics-nominal.png b/dev/reference/figures/metrics-nominal.png new file mode 100644 index 0000000000000000000000000000000000000000..9991d6d0284f1a9ed11c28e808eb36c78baeea45 GIT binary patch literal 82676 zcmeFXcTiN@)-Q^pAfTWKsN^IdIp?4P0wOtQ2@NzE8k(j_Dk@2G&XT3cp~=u9AQ?oU zktRwMkZc1@d<)%spL6QnSFgURd;hpqXIU1#)|zX~ImaCFH%6$Yy3!3|YGOP*yc;UY z3fg#h*M8yQU9P=$8F=#SWmgXHXBDn%=&5b#&FJRgY72G(F?#yAffzwBuq_@Q3=^Sc zLW<9H?c{($=JI7(@yJb<)f@1WLC%7S0O1S;=v_lv5!&*Vd-QkeHNWUO)TR?@>MRgw zgL9(pswmNlj}`Bf1!sm5S}VUQ?|)TECiOl@X=p$!v^emFF{^0BuLm{DaMb!Vp~orx zA}Q-&2Ka!%FS<hNYl`b8hY3$z?gq`YmYiz7o4N#*qHck9)k_05p8BCvLAZ;PwdPtt zC~t`$Qc!w_N$Z+co_dl8PXcr`gqh-tvX!#EvdYO6?5QE12x>1VfceB)uK5z_I}gWB z$<kdsyz6pcd3jA0dHKIL4Qw?%!GO?Er-v%jBsTh?eYPr53lX#SWo4r5H#2Dw(O0hB z|NKz1GB=#K%#8cFyn!M`iLg|Tp&^B@1vdxBx>Z|Qv&tp2gM_VtgMp0mGsv=>-C&0A z3Mrlgy9BcqFM{-8X2nBVnx7gP>MHqhaJ*Z0@u}Zm<{=fc=y(@NN?36n8OXQgf~SBn z_lkBntv*Ll-JqAtBEciS;V#LSl}5P3gtz^)OxY9<PpdaggXgA-K*>YZ>-eu>F%)*a z?)emjeLJ!#YJS`#^ZI!4y5h9%Y<P$yCJnwt+s8M?0zxugOTWB{CnDJ)>*EvWbF=V~ z6TY*;O;zl@Rk<HFXKF(JhH>pZw4Zkyy2G?zFj~a-rJMy8ECePsoAyMLaT)4Wbud+a zm1u9C?oFX(>*Tz7Ks3UV%ZPq`81#a@I{0qBjml&4#r+RVVp&%q1fGo0dH2ho7)4); zD<{+3E1Me;y7I^=<U8@BLL%aCJ2%%KHHTy|+%2q_V9um1-hP9&3o&|jH87GT5tncT zXY5gV&iTv2ZfAnn{~3~lOP0urxP$7=s}AK5Jtk=XP$-Gcw|6|fR*Iw(A@?i49`Z0o zE|CPCE*%+Kz7^-tx3ox2k`uOYQcZk8SM-sPxN2U05!YtiKx=g3#`_J_8lA$Q*FK~{ z%8*JD#c-1&OA6h7hppp*r)&uEiS?4Tdl-k9v1L@+4kReQ`CbPv?u~m`CCuuQX|#vG zw7l+3I=-mee7(!BpY$!XB?Y;CynN{eDQiVRP!A)`HOV9Dqq&S`_G@l0?%G_Ib<^aR zXNk3uc?^rdSDe!bIb;lkrK3uc$cUNFy^<3w=pMc;snZl?CX{|7PWJkxDU2^Bf}GS> z^@;`~%VDrLX^H7wSKgAZXMsaxm9zKQY(7!Gv*~6rbMB74t9+gD?aiQ&+WJeILB8L= zZ?6#+3k`mhdAqzbow*(VMFpl%ul|F1DwCCi>YM^}<;Sa|M`5OU=gA~f#14(~XZ2#b z!M*jQE_k$VcYXwi9zPx`iv~TIPqck=7PmiZyWKvb`7I`+o5LvSr&2A>Cl8T<Sg!~Z z9+z8IqIW!UYc*<@kvq0KjrON^Y7eOGpt2;a6#Yp7rI!j5!IOkucI?1ZT5C%;H)eM0 zk4M)T8c9oNDs7Vm%4_JS_V^W27n&FH?Y3S%6J8agu3!r<2)~+Qbk&n>>eiuWZS=Z$ z?B>tKpO;^F2F3)o&UAF|-%uz#l@dEA!)w;KPD)CT8+?<A_Z%hVcbe7;5xrjT#ewfu z7Rq(&POpS1oatlvYihrdU*FF_q)u~w=`U%vF9$H*eEo{!CYkm}g}eS#rYlzbl>Jxs z9ujfj_q9=1Tyi3q{K#8zhZgU~9EbST!B^pPg7u`EpT<URn7;_vCiQ!A^Yv@ZThuvG z$&861GV|~FZ!)mnBvP`9p<unyqwqT9N=j(IQg3uZH-Qb8cEnGm)M%Y<pS;@*5z-3q znDK6Q3VA(Ep3!T_$4c4P<3rzO@BFwTd^_V6RW`|&{2!7v#M43O+`=#6KfG#4{9lqj z$J6=j$*<FY$@)1rH?eA`Aisbp4OeGxC1(kh<J%z|C;AQ6lEIx>y?wPF-?WVhc6qR~ zaISji{vm<i9rLKVd4ru`nA};!!~1v1e-dS05oJn=R!M%!!4!WRdXx0_;79MGwCzY` z*0r0Zp_|)g&zNj)+ucmMDMr2?!XBE@(baK@JTy<EirptJN10apQqk+0%$i`&SDtT1 z)sEihmDuDgmQduyBUp7g^*IeNrJkm%VPA#vhits6?oY(gDlW$q#wEqn#ijKmbx$u2 zQQQa<=}aTn=qc#WEy*3#9@N~_n&IH$N_*hT(ZxX$zrtCrGgIiNj?3OvqEw@lqD<_C zQHJ(vCuDH4>$wy+XKNSf>&iKBB40Gd8>`8v`9C~;|5V3I7gW5fyYfN*1G?B;%fIM} zezmSOLQR)To6A7z15~9{BcsG6e^7hn!;pTtrmHrpNcR1sk-Me0@?#1s6|Cz%`$W*I z#TF#kzGu^byj&I?7)gx+-yKN*oZ}AO15bkO?5B+tcpQ0Z^=tK0(B`G2r2)=r&UZEx zHozE4Ojk{lCqEq*U9EcApkM~Rq?o7dD)p+M7p^hf%h9X+N9#CE_DW&bIM-P5NK@H? zN>&-ehpb%GW7MPb)=MtW?QUB0k3ElTyBE}Y%b2D4iJMY<q&WY4B$I=^-CalVcDFRk zrtdFeDzz$Ny6szr!MU>5bk=eUP77n7yCQtm?Ac`5F2{|ru|LZkL9RZ7^ZL`ZB+R%v z*i&0^3QRj-ERQDoG2Cq>-%195wn}uSTV`6eVwyXfXFqi?f3hl2jOe6Ot-9YX*ym9S zN?Y>fax7*oexrXwUj_|FJFPRXQ?L7q>e5)GJrLCs9d`<?y*^1h8Bv?Gp($n1#N9;P zq`Lp1C9UOSK(7Ca@+ak}m`^c!EUuCt#QEun=^SY3CD+8?N>NFPinr2&1US7$!?c`p zTO}GKmn5CN8>NgPo<Dxf4-e#8NKHyWGPu&qBeEljsD<pFEhHQ8?(_O3sqxmCg3E4| zk(im7q8b~TLZ=<3;nO@*BU4<{u7Wjphu__Q2cf@078oT#uOuT2Z}g#p(Qlvh^|f(z zIo25r8Ln5JR8Dru?u+g%!(6eOyV=KiIH#S}T@mckp*~J_SLk2?x3TT}xdvn_bii+M zB!vq<B|Aph`Bd3fsuMiF)OIQV65Y$xmqagVE;C$7xe6i}yMiTbA{e?Rd++fzA`+up z9&yec%T>h_r`em?)sIcubwAC25`8U0Tt{*zSWM~tt?XOH%$thFPl#D)avoAhIp~d^ z+OPk3Fsx)1A}}mKDWGAk)HV3dIXt&hEo_qRfOkeN!02T9O4=2b$ErET^M@U49S16x zRJh_AW0md?J#9_4ie^>@Kf9mLR<Nl|clVCC%Rv3+PT7L$=h@`vGM1Rvd$(_}Evrk5 zsyLI|eQgtdY^;shtl>-NP^YwG%*ZfLnHd7Pk9$~bpTLTJTKrtko}N`--+GsDPbOt1 zrC89n+ON8|`sfOG8$ny~&*+~66I+lR59spsWws^s(u1Y^B@rT#Hxi%FJ)h$W)ix<6 z!k;Rb=|~rrq<rNgy}i|~w9CFIMrGf|fP;ocEZ*s9f1LFx>=T*J?LzwW0s%GgQvb&3 zU!5aalvgQFMe(!RGKK|5JeQkO$m|5{oHHW&)w8AryZi=UNlZh6RyI?WQj&zW4y+32 z6FOa2*auwApVvI!eAR`za;~kDW6aFk(D23a`?BI8b6ZYir!`f%#Yt(r0X)0EsHFL4 zANgi12`ci;Yq~s0RVWe44Kb+&OSm_<;UXwoxuB(^hDJ5JJBScHZ1HNbssYMO2t0uE zNZK86eBgNBu?hXA+|ksvG5~kPM^hzgTE1QWp;)-Y)D#!M;<}Ka7Lev3M&ryoc>)f9 z-d9@MX0l#fg6OTgRY&RK+bHAiJL&pv7U6yOC&%QDSD~BTc-c3z(y`64v8Ef|e8rC( zSw&iWKW{ZOi#>O>E|uRvOJ+1y`9IIHncX`DAG^|ClltNj?Ow4Ty_DVCUelhBRX+W7 zboRF7ZQ1Ae&w~YD3KDJ0m)n(Sqkc*|z-#7{_bWkN54!HqGDvHATFvGCu$e9D*NHM= zLr%$L_#GKLK6C6!sSsKjB*>7N{_G^>K6pGeyqCF>#>>wuZ0gx$IJ=1nIP@N$CG)QL z8ny61jlw1tJRmFI1{Vzc%F`RE-LOAJ_L1S^Eal!a%JuH+%ZF)Zb+9_P%gRiESK;>H zN)u%3)@H(t0m|a+<MyL5hT6bUuLdtV-@Drtr&?3FO;HU|7ScU|yZOFtI%^}lc5Fik z-*n%|eFCg`fb6Ev0m*RqkpKNahZ7^**X5;b)XAB{md55*!2JM`LwFC(shHGkW3%(U z5PH})@7DCnK|aGTxt~Yfhu`)etd3L<>7b%pOO73mkvNlYb-qD9K|#!s$t=f|AD=4< z3F5(8ZsR!+;|arLbKk{fo!eZE$ua_6J7MRRW9bc}!9#AJ%M_5EO;25UJ`07NygG7N zxD--w`R3N8F9OzY*S{F4LCfN)n57AwLUE~?Nu+0LF<BCUA;&9J%6euWhc-}nKVSKQ z>h!n`fxHS12J$CEwP%vnuFkxcHm+76UYN5RkXYg2Nz1|9EUg_uo{Ux?JFtrk^Hvjz znGtLw!)zd;#;@ik53&a<!#zMcaCKd4xTCd%4YQmqu{2B)Am9x0v}A-iJGnq4VKU6W z=}H2>FCOzTGyWdp=_tc&sHVv%@9F_!6y_D?<>yg^fuVxTvc!zi9yYd;+6qtpA_3gV zFxz{2x=Hf!d3$^FdJFNodf4#^NJvQV@eA?^3i1FWcpyG5o|Z5k7YNG*i9a+HKoDyW zu$w2?)rIkbrlpmum!}LfGw`1A@AWymsj2-#-Uad(7XW_n!7SbQ1bF%RoSpgp{R@bv zA`~F<*Mk0+UqEz$G?Y&p1abB9um&kYK`x#w|4u?lMNRV`3Kv_l13SC@-W0&R|Cr3i z`k!%bULH=r$Jki&ft*0j08t3AK7s!j?+Ld37qk9jZ5MBTZ|C0^0!;T0{r?#K_uhXG z21co=Nh-KndtFSbq9DV3v0h0VS8K41<nNzC!lGhU5|)BI;zA<AJi@lZ!aP<&BH}y} zAR#MTO9^pt5nJJZBc<X3@w9ZY23?Q>i1UI0I#yP~AW>_9i?4-*c!WV9YaT0GYa1S6 zAxkR>5pgkFaS7{xBcb5|2Bgx`>EBm%LCOXoB`j_$E+8yq#Um~%B*G(XDIvlm&Tj?c z5wH>!vf;P2va}Ww_)Y5K93&rUs>m=4^78-diKdgKr>(1pvkbGEr8T35&cEL1f}KG+ zo|YF}6A%;P7Zei_6cG`Z5EK>_{nsFUkOu^ixC^2J{JcWqzu(wcOFjWeS^|;-cDA$w z@wvI!{eE$AE|S1*0M1%o2r5AN_j6z~lJXuPOHWr1T~}8p8Rm<{GG37U=d~K3Pd1jG zmI{`hAYe9rL19S&en}yIU498kett<2AufIa;K#qlyV`(leg6MOUuX}b^q(_V219`P zeSSasqf$B`_dj=kZk@otb%~Mjw^m46TL0Ms#1aa!`8_W{>(47|drKEP5MVz3lGneV zgZ~$@0OA)DwXznp;o%qO7vmA;w-Mp7<QKQ*5fB!$w6(I9uoV{lv#WooLtJe=y)8XJ zkL&<G0=xp`^Ea;;IsP2sga2HNw>{|Md;pH|2=Md#i(?{xag6V;9rImSjlZ`n&G-Le ziuCUR|7K=@@BTak%okt{`TjA7e=!YM@c+r{ufzC1IRzu*e<%4L@%O*%`Y*fwM;!Pc z4gRlm{g++;BM$tJ2LD&O{<pD<_+JSX$OQ<4yn%G7n$!0tkZKXRDH}oX@OW7+{$GOS z%RzyMgq|vDiiAIi>9665!D-I1z@vMfiiV!@uFe-xINtBr5M;yX4feEWyoe1AUXtFy z!(+r#QFx>a!~C3{^qTL+ys)$__eYK0dKdC>?HV^Tf8<Bz%L-Q>0{^;iy<ovp!lS#i zcI}J8+<6Xq6oFFbByxkmvcg+zn}&<jImZe)PqA3+*d<`@KbK2epz=le?awtu{L19- z-+1-+uGIZ=<0f>?<)5$SY;R8e^U=ov#`>#&k9(2!$c^Oh8$7X~IjVndTnO5D{<)dD z{@;o4@czrX-vaS}tuQR0Sle=qxT*B?I^9oBPH4LXh%RR%5LMj8h&4}1EsFapk5f%c zH=xHj+{CJ<xw$z#XJKy@cc1o&cPy(Gh1T6a`tXAA*En!ln>Wz*mLs0Ni=H{v1ncRw zn7g88!_R5Dc5JJeTHkCyr83m(CQJ7Zq=)RP@2x_dslpXAS9Z?_SN6WU!43wnR+Y&^ zE6j(S_f@KlOYJ-UBCfROCREqKxC(6&3El-r#IpKi&aTH7VBtJ0CK>95+<l!^kACI6 zaj7dlU2+qj`hk2_IMQSDICDmmL^Nw?47KQUw!aU}c!{gx_HD^Lx^qj%weq%vrRp&| z^Iyi|BR;%j4h>BIj1ALtjyMg-0vXYE86`$^El4%_RYs;jTSi|}l`Q<M`@li0VY9xp z^rpF0)T+10C}u>sWxRJj%*jhKqqxQ4FZ*x@n?=|#h(HhoJ<43k>`v58T(S|o7xBb9 z@!E2oPx7zrxq=aJ?&Tmk-Y{lJESk<>F^5C?yTB`xaoeV8=sBwG`NOI|q~?CogO<^r zl00EK!)#U?<-W#9S8>`+Xw$)g_fB9OYXfQme64PFYoOv$Y60_e&BP`Dz*n136t#rY z21Z}Bc{DUH%q2RNk5Y73afjtpac5;^Js=-{;;6T2o5}TOin3n$D|KNxUs|<U6tpP1 zD(|+<ZXGP=mEru(c6U8A;_$<Ax~rT}vt8~}TL(pWQ9lp1W-%7Atl%RVRzDPX-&bL? z^6`1pmJ|V)H=}WKBiZAkK?1m>>w~Yk@KBL#@M%#*bXhx_)^mgn+vmNhpP8NAa$4e& z5FsSngh@LDGR^pK&5IB2o!lzZb&T$$A)@$00B^2s`dq9Ife@TM%^I-v$#fK=ui`c- z)yMF4{tEes|HkjjIt`RL0p#<!rN{p)#<_fPx11@ke5DbAkhIRsJZMzcSd;URoz+oN zf0n){suNL@Tmb#ddV_D<PU45r+W4yP;quS3p!~lqzS!cKol8m#&rT23dQ@jg!3LD; zocZk?GsxB2$&IeAE@+cpR4l7XEGr)$-^;z{i|V6ozoIjmQ7A_S^0Wb4mDqTY1a(C1 zWa+m_8^K$%=%bU9^k-~qTh-jYzP@qi-on)2p8Hfq#R!K=!;*1FT)8`KAy|?=%F#|M zd6G?qIg&<EOHZ%7ynJmdj+N+SFvUlr{d>jiRuV`eLw$T5S%1Ram)Mt&J(wr&mkGZu zdb`$)h~fbwacCmS)zy`$`Ue654-CBHs~L<)OBCjr?$_GN;jsDc4SJ3B$l-W8QZu_% z+}TqSnmgRh9bgr_hcoTt3w&kY`6O`>(DAb~UWmnFmHyyqf)*<>W$^54*;ocgm_lAN zf^(-$&Rv!{`}0Djak@<`B6Lf_s(gIpEQgm3Tu}Yt&iq=zNKrp<yf&cgP6g<7WU3$& zZYI(vLa=l+uqwc{1i={-<=}to&g9mCg}J#jPW<lm8WIYeAw{QEWweo>IBkE1Q=Q2U z&BlA9L>W0bLUnPjK5dHU8&F27*c&Q#!=@o@2Bq0Knm2;mqo0_xaJ{jqD~Yu=bwJJb zaaY%J@{kh#kr>a`%?#X)Y>qH#)a(W{?R@B{dSS<qMk{3D&`lf`=_O5XW?37ZVV|9< zURXWhL>e3<+F&V|5;B`6fREF7b<ZAmbW}yU@N^G11$n8cRc~t?vMxzWikTe|zR_>A z=Mk2}KP1{GIKH(c2kw719QFt%llqM9tbq;r#zr=4d{Dn=I#h1Dr2m~beH2pld#Af8 zh2nEHKnOBVcKO32IC^X&u4&{<yBN^e_gD2JZ#z$lM|R4Z`LGvf76x>>H&1T#TjRpB zQU;loq<xYKR(D{(@}SPW^}Eyk4`Kw7m$xizp-!S{m^rn~m9hng$*^jXz69tLCg`*K zPgl<N^`!MgiWyT_*bZQh9gxIeH~r!Sxm<jcgfR!bG>hf$f4KDGn9K#R#zWm6QfIIz zDrHCnHh8OpQof3yuHQeCe2Q2?W&A3f`NrC}Q=cG{<%zSRIrI+fdiu}%w!wx&tS3!R zwJ416nV43o1h4yx(|%ehGg?m)sRum?F(Isb+?&SrdLi@^0YOjyoWAq)t!(>q*BlN( zvT^2my)<!^*z}oaDY&zNwo*)A$$*kjJ*bp#t#%J5z5PD(y>YIm%k(1JJm8SNrb)YL zPJg}qub}PVe4tc3a=>!E$0mnIj<Srq&%FFjy|bR4PiC4bizp`aE~S59VEO69@$s<_ zYC<SO-O)RFrJ;dB&nOXzpJa9VQ7E&t7BQBEJvwq&ZYqwxPJqAHrKt5fDlL2o!+TJg z3ebZ|<OJ|gg=4Zd4vx27F=L7O(Yv-X#egJ%jGljeGd_iBydWINqAzmtwqpV?lP?vr ziOY53dKOLZVah6r$r9s1fCy2?uAapLM-RIKm(6nIdQz*p!reEDgiM;9J?FRAQm;vr zYgV&2;6}Em7~J@MCNF}$&S$5{Exri`T~XAcP>d-kI&{#AWqqk5R<baxMS(p&<p|q9 zZV|;Z&>OQC{#u}8=*_Ioc45+yQDDINNDl#ub;1sI@nUXh=LqWvO+nqtg54txUSk-D z(ea6Jr%eJq&|Ynp|89pmK?CY4L7G-j)ZI3V3cRD3%~ntxZSQ+Ct6ycL2ElnN;tRCO zkh{d<mQJTTcR@jqspbJ+p<U6Qa!IG2-<571Q^1X!#<dJ4AFmO0xSQ%Fh`0!34F$Jn z8;6$Uo@4~dp~hD|M>Q;q(volI)y#(IkW+TSW!GF39Zc58;Vqd1wl7QL5~v?<vq$Q~ zsKVbs0fD*^EI<5g<G^>1FuXHo_|BZ^;!l-Q`_M%FQ<PxafO6<zrLsxsXD>-Q_p`I3 zc`|XM_PzPYy<fj{^~TCAP&Ip5Lo0w;#z>=v?H3yg<3V6_eB1N)>WNk60jULmd1UUA zla;Lzx^>k==C;r_RafQV^R&bLJfF;}u~>^LVH3)omrtto!0z>{p47WBkN;?j^lPQ$ zOsF)yyuRPlSk@uPImQKjYva4yhttorq59*<>P;JI`eLcdfKnwBY#$cmn_7@Iund!? zkLsFsVNc3(sN~L8rt;bCS{Rn3D&Y|YwLElY;vN??E#y}HV1D-9?Pe06(67nF`!(jN zxO`g3r{JTsg&ljg_qBw3#||$YTdyV(`ov=dU0W8TwV8;&IUoi1`r45M7&q~F60%0( z<iuO(kdS>E$CU8^>r$yT9Cwnc<BO-IcY@p7sPBFK+F@~Gx(cjjspltD%5mPFP;mUm z2)as0_4&#<-^4iF=<#@Gqo~+=iBB#1fj;JG0@@}t$6lH~oEa`otFZ1M^J8){tyX>y z`keO=Os_Sm;T&g$25r2xYVmVUQ3%vG?(?BgT*eZJuG&e^{$Uxt{>r1W>ft+TUxmH= z@q1_oY6z73Z$YrWOfRJby10<pGLCM2v?ITMR*2SvPKCoIl&KYC&I!?{hDx6`uDgIA z-8tXlqnHvT10t9b!ms3}El|k2Z%2B0OCBkpl*9Q^j|NlTr0)sL{PiFzA;%YO)H+(> z#aP1A0F;>*=>r_VYVPNlB;S79wp1hAm@mW#<mPBWp2T}s8M_S{Ri0NS%Zf&hS@8D@ zJN3q;iG4t1*0U$wC+JxGqnvy8_Ec3O;ksCUf6;5>6tTV_Y_%Oa-NP(3Wy_(Z;_cGo zwIu3P%m%b?$wM?vjV+G5uVk!uDrmntdgR*%^o)RAy51}CFOxNA=|#Vy9o1!h*rnB9 zhq($7|0X%ykq*uMtaY;2coM7=c<-)PwDFa?8s!)3Yc=1!AaZ5=u_-KtSL#yD&)xv* zbiyBh04@e0P&JX`Ce^vfk8_Fl$lPuS9QQ?u{II$~Gg`$K>Gd?&=;NrPuttyGYiLu3 z+*c>_rj)14U=#dA+Z`!^`X$bn%P&i=ejG}7*gtLKsb+0Ty|tHQOqJZmV*nOQBKAD} z>cl^GT_oxU9QjEOzeL~pTM@Ku+bGgw=AV6B1Yx7MW+t3OX-UV|v4DjQKpmV1$YvSn z<&(bMJn-Ex5pkzAL2<I0@7Haj8*h($EA*GY@ec@q%Sw9eZ^cN<$QYHY`#tGKCZyB? zx>$BwR4y2udQPG-`=sdvFzZ{cM8?!%d;=kEo4}pI;GUms*1_>9t4d~OX6p)kV9L{9 zw`g|D(O|6{G>`AJ(hFbD2}ED!K0U&6Rb0ET!rV$(2ufrp=~t~DBqpBNevHk*zU-}{ zu<-W{4D?Re*!s1zbIV)-sZkP2FtMj1WRUBSxPdx5>Pw(DHk;Wx$Wful=h$8(3l1tA z^Fs177Y3Xj=}mrRQ_0CVw-Jo7DDZ>BCAUtO(#ZY<8ZV^wW|f-e#3T2lhxVGe`%VMS zcXldgx6I_X6}4DZ1Zhb-&reT}>l(y{UX}!kDY4URbkGvrm>To(d1YcHu6La4nEaK& z*M5x}IOe6TgX0Tl)^ObgFyuQhY3hi6h7UobR*Z)m8yk<uT@uA^c{SC}uMw#WH%e*< z0;@Vb8!xvoAlE2${-r+E4sAL;JiO^#Ag6Jzn@{3on8We)RlF5M1LvNf=`+r6_h~G% zqT#kiD7h=;=%!I~W976<saj<hOj<Mu0y3gf%+AcTcT<m&_TT#}Kyb4=+p933?{Y`Y zTJR<{O>er11BPXtaT(ynLeTP>HqVoY(&5YFQRfG!b8sN|X!-2G(ev#rRYcQZ-$CEr zl*ob_#-(l#cd{~+)ndLyu0;sR%KYwCU2ks5*pX$Fd3l020J>f~>0A!T6MPTn*J5sA z_f{0)X)9aX^g+j=aV4ZQ+(=I^clbwF6oXIZ9YrSFaS@DsYQe(JxAU~q01FC5;8SQ5 zyE^80`ENlTJPBs~;VJ1(Yi9%nLdr_f%7PJW`Wlw7jvnVRSjG`H;Y3A#dCH|uF7WK+ zZ57;#@sgNLrQt8QOrYj>>Z)YhJnp_Mxxk}4^w)cedTi$5L+#Q)oNK~ORhYgn#z6YV zz#S}}CEeE@7yq3hH?-P+UG-#(>seVT6iw!mgJ2iB$Tkjm%yDAg^F+!gdqZ*<i?TpO z9QL{yNuU|7Ka;AzPuHN<DF$vuIe>+4SUlEY-CbWVUbl9&>mSh3Ug#C-(_tJ46tqK3 z$5nCfNsFEH>WBHl;p7Y?v=L8?X^xdJw2=I^Z9gZ2rvnJU?c#^OmY#1Yh*2xlS@#MZ z!h~+^DQb;wKpnjkKt_omE2gKzb1-s<U`lgKONRCU3iqd*0<WItfMJUII>FjGz#z%m z@+sA2Z&T?Q0_SKTE8P*Ggs^80KZBf$RYqpv6w6KnTu<$L^#;O7dSap-#rWh-tfhH@ zb4Z-rfHuu;1)d&(2q=520Os1A$VPZ=Pg5dJu<HgJ%Lbh#YMGaP%HDUE^4s;WUZnPA z6-0s>Jtwj`#_)l4N!2k}K!8gl*Sg)BDYnl`W4Kl4{u#Kz)WN&)$$)WcLDjum493TY zhu>W4Y;W$Ip2|qW0`CXo|2~Y5boJ*TBIIy80&!C8<nyF%K^*;=*39>IxB`~KBTw9q zJwv_g<alpy4_nwE@d-c=N>L}(ea{+pH#?%!pt!S!Ydm@2)D@gZgETGaGGqNcOUXCM zC)uaKJkv+kLf`g*6nNn3*gIj@Iul3R&`aJzESvR9qYjTK6w%#i5pE)TQSo)GWJPaX zxRVjd(8`Kk_z{S0wW})_+tqUJDoovgIB^mcn*DB{p`#KTvZdo9N+Z-Tc=qbkKk_8D zb_N95cW0mTyS22-W~dJgYk1upnMxCJvW`3}4)R=<re9aT2$C|b=qgN}?ExoS(Bmn2 zIvDd5l3v5DD%4nO$gNIWF2*-Y*`B6eD0ofs?)4|RG19-RD(_w=n9pjQcBz{bQMIM* zIs=E;SyDCN{9M?yv?y#{%zfO|Mn>NWn+WxtEQsZuWaW#<qWPb#NWQuOSZUiJYjhF< z(Vtk0E$IzFl~d9?w%mL$u>n;y3-I@+10<&*<tkV(#c(}oUuoh!U6fKah+m_f!PlgG zJg99Bi$#vFpPt~VCpS_reG-o}=})aH=vn$JReM3`e%{6d*$lbsK*93P3a^g44-21O z$FjwR?pyN&Jf~Ci_xIN$e03-MO#=f~xL*gbCzua_#*4_oAY4L<C0U9+RnH0<?oJ5n ztmc&9xB6rj+l{Sbm-lcJe1U=~$-^-~ij(pTh?(z(@P%2mnD^c2)V7|_y1=i#(k9gn z;)K}a)9uA+9g=hs^Qo9bNh-Q`)-4oAN<g-@fG$C_G1FvCX_j((8F=f3nP=Tasm=?4 zqEf?v{9Twj#cmSrk=^jA)QC^=Jb*}}<PN}QAV;Sv5G2tyN^CLLD7Ns()bCD7b1$jl zKHUH^l^i1S;><)4SdAB=pR)h;g;%yBMDAs<5MLhlnbaTKGL>?CzMOcN9DEO-BkU7x zg(!HKdwm8C*vJ;D>W3wLN#NWQaO(cC|Ar#A$3~k%@dAbmjnQ;wcc+c&T1Zpg;|SZ8 z7VFX;5Ehfu@EWcr<|cBd%{rXm+1uTd20~<`#M9FQAm%@>9pu*9J6Kv$B&l|;n{=s5 zTfvQt=;-MIuQG>LX8G%e;1a;Js9E2zVyJ|Hqc|<#C+=HkuH*=YafyW6CxQ-I*enZ2 zRALoKNP*=|II+coP_ukgB<JMivw-@Zo#1o^iV%o^vy+s9B75Ib9FP-hgrXt=B&aUO zJP(WrxS=Xlw*N_iwkyY7Gq8Ggt7*DS+-Gk@$5>5lLB?t`DWu9k54*GTJynkld+hJE zY`-#lH_hHnGGh>k_!YRHPHKi%rj&~S8Ndc~LA(s%Ae$9#vexjogmq!Dc%;GdO}F}4 zRn%uqXuE>jA5He`z?|Sui^rZ$hTgl#-7`H6d6Uj&(-yF41D_kY_7(?bUpHI1ewO<R zWc+AMpMbMyTJ(wFZGF|?X!xk`p(T|lL|dzL$>W|mANBp`BSg1>I8t<{n%mJV7e8FB zTjXtg7>1h8(EG9aBsC=|U%>a(pRNxt$8}z}{cy)+S(mmeVpq~{Z^8)*-=qIpmErLP zKq4kL)KPTy_xbbSsad|GVHV+z0>ure)xN!xpMM)V>qfxPb-tgmtp-cNlM9NxV0FVY zIUL5N`pZ$d91b;cOCCOu*+U>kJA(k(N2wfaL_M_tITHGLVP`h*-2bRjzgQ3UkV7SQ z%pw2OT{JDF@A+BLuPG<d?@2=RQ7E5*p4+OOKz<P<Z0GdMB_{c~aT+zx<BB`6D?W!d zZLBJ(1R@%8AMASu;G^@<_ow2lnl@#8@LpsHopCYFNfbh-oIwIB``Z-@oB~;Z&a>KU zrms1}aweQ=C!9=KJgr_diAF~kth&@e-dvd%pV@#iNfAk)Dal{9EO~%iyZLz6qTcDM zv)vvKLEIIQHPBYls+rtyP;$-XppNP)%ihT0cuZV+w+LkqCoqrvKrJRlFuir)-DI|5 z)5P5eZJK6LsC~WUu^MU?+AOQediW06B>I%q0dPtr*X=Tw-XnP$>gh(G5SvNex@0^g z(QjKF&A(_PD8OP@W|FtQ_gW>k9-2c?H|Qn)M;)aL&Sgi()B*QyQ)ON*CnpEQqyea; z*^`^9qWR#X-Cgpk_olR6KsF#vk3yjUAJI?@<lf_&kM~(}9?ZA%FfcHvdGG%El^5X= zR-OPGoHz$U=kf@l#?OI3x_*XR$%+;p0t`|W0<l<8aHu)5+F;}>k!ckTngD<?24&UQ z#nsgZLlAeX<xJ#lQnDAJfbvWp92`_}FV#Wj#nodMaX5)S!9a&@89zV2c+i8mN-Y&B zgUg7dT)iw|wb_{&X*(<WO&qSl46US<x6vyiOCJ!ytjoez%B^a252Wt%<U9VldN()? zYfQq33%7ttj*Buff7ssLW&WxVFB8KEB-J@Tj<~{8bkx=ZMg~TNWoSvO6ce!bBNxOY zm%vS;WUdqaLB=E+xsz>9qBNwz%NvYgIRU7xSryc*Myx_{+WfJ3`*+QF;f=MDPK}yx zyR@T@TCOo??=&?E5S*2S!~Obt`7XZyq)nGjv7qBuB-=8*RX?FkYZj)(kN&CEdD1&C zQ?<q=+x|K}*aCPkdT9XKG<Noh=IGCX0VOG*WH-3Au#gL2oq-cBb<1M?y;bSzg_v4k zBdm;n4|B6LdYo>drw2fB0nsWU3*tkN9jz?DTAdz=WDO0>0y`N>iuil<#~_yaH{n2W zB6s-1s(#;+V4t8i+21egfvEH2{Y4*H`Y0M_LHejrr+?~<a<(qtUxno)s~3Wdjsg=4 zR$<cWBk>e}Vi(~1fGc%w?(5j2g#~LA0J{6)x;VmsLml}A6c{T0sXSg&a3h&5<?a)Y zRQU_$#KZe?`QM9?@6MyI{Jk8!ddmOrvgb(>E8dGOtwd1aNN5g67_;`@OTkz!Q@K5u z+^7Xy_TkP#IglHzm!kmNjmt<|z{G<9uirH7qD7JSc>}X`06^41Z4X~_Uh<1t^~9`W z&yL3Jh2h&*9)!(fu?e0Z{Q(y#nE`Dw)7l9<IovrrQ?Gi#dw~yJd1X>BE<ugeG~jvr z9SHyR^yc|gV*MtU>mN^U=$|N|W>LRhDntNL4D0bkOKWS{_<9ZpK~S4~g1OcQz`^Fu zqFP(EZ3pd9vvSYvfzl5Ut^?*`A`B{-k(89A#7hJ+A|(jI!ez3CkOcvf8DWoe*bLqe z+q2dJhC)}&RAV|RI|PBq%*q<7+Cw1fHHP0Dvvj0|x0pxK0>>s!*GE1--*K}?zS9;s zw`Xh_>g*T7%+Td-l(?{S=qnmr$PxD3GPPhf&{N<n`Jzm8N9K#%`3cL}!s=@1GgkzH zbJ(WS-D%*#J=etZookC9t&9@GJFUcN38+5FX^(Gy+nPmn2AVeBYv2TUhz0^Hh6~i6 z%)DlFU&%4t#uTB5T90>jeG`IH)asCu9|!WB&?b%40>IO1{la1?za5Ru3@~KZSU}1T zhicOUD@fSmPLD{;jS_P%hhdb4fm-%>=o}UA83hwOi%f*P&XBt!5CDy-qIh?Ceong_ zXs$xRzzKqk*i@<}H(GID6tzND+Fh?{_ts%?RR$zmTU$Pv-jW$(4xmzh06y2<Uh=?+ z_w25Z^WkOzsB>duz@pKHiS-lQ+(UTZog9#iuz8BE9hbUXqua5pJr~J)AsBHrO#@fI zFpY`;$_|}QpUNMr|J@4^Dm_HxVkR-+zIJe2Jcdv-Fm2`l97KsX#K*$-+p%u>_-#I7 zY4z6+J-;ImM?q%d_Y9iOl$wX_Ys|}~G6r{G&4}B#%bOLnJe}DI@bJ#ACEoAo?7Gzv zC_Tvd%Fh6pudaR#u&C|QF{OLU;mY6SViEJVZ-G+-+J}Aa;V3-9Vz~-;rR8WzgWJDn z#~PUB0tJeDsPa>p>}w7Op|Wyvoq<5U8y4pvng&FnDfkDa__|TsqF=2C{hPG}4@Wmk z-(%2K`_RCH7Avh;280=<Sv?>fgj?BS6nZjCdZDgANOqC|-2&wOJz-!BsdwRqHW9l^ zB<A($nO$}jcV~Z4U0t24c)IDv>z8wrob2PzOg`GlW*sN}ijlWvU6`PW8`O&w*1EsN z*g(%z#Nf8MP->^YejL(#qrte3(A5Msb*QsAy1&i`8^0%?V*sd@a8wg_Hp~*R6ExKR zx4A3=CyBy;6(M?&#{S*7<SoVCY0;KwG!0bUT)267nf>z~wR6DJ)YndvlG7OHZ&qAJ zF^+W(zwRS+$kD9mly?i_eN|{L3sdk0t~)fyLA$#T98b4Fo;_W;xVLs6{l(Rj$<8NM zuJWWtxqUCu&>)+9rFz<>bCG9!^4Ho{c>~!Ts?BUZ#6hsO&r`_KWV~KZPcJQTZ13}) zv4vMtn(-msnyC!cdVk(WIs&L1?<mG^sLL!hGOTv9|H@323B@qg&}H_SfuV$9HE-Cw ze;yQ~0l^4o^X@44tVVi{6eggTzQ-eYinYU~yf&1^d4EZhD{9epWtKX$mj%}LjW;4s zY7^6m>hbU%9$@btowoi^x=30`gQbVY&h%-!068YsFtquy1G^~E5h-cKuQYhcSGM{U zjsP(fngMVCJ3&CAH0@S5Nf*@x!1pX{Djwpr9ehz;xhXFcwG7GFm0C^j#DY?$s{3rK zRYr8znSDhm2q`CcY7<NvVmo`eIhh}n+>y!}lA#2gHZJP^h@9Krl1EVl=h1DQ<zolr zSHYawtzc6a5T_Xob9B}?f*TW%x<F){|Dt(2nm{w}-F^itViJg04Uzz8>f4Y<gHvNE z*6(B^q1k*`!G3Ah!o-AOv^oJj_yJ`wq{9;pq?ofIt`?z>*Hz4YG<nv|8o;G&nF4U^ zCy<DDhW5zkrvYbbw4MEYJE8TMwcHGdauc;W6(Rt|cW&ah_J03h+)XawjaqsXQoCMI z*CwAkE;9|8NNv?FtX8U|OOk*qx0NbLNJJwib@a?=927^5_1a0aVMf?@1);h(`08>f ztMtn-iw@p8Bq;eYmsr+N<ScWVQmOwJ$(KO6ances>zxtgm{mGix2-+Qk?j!v5<^t7 zVP$k+y>Gzhc@Pf{kfsN=iayXEX*=2`_mDN#BlcO8ICjq0N*WunBWWmvrII4$$vrIW z4F)tyZY6Ch`@%aN5{8W_h1?xfW02%M9k--Brxj9X3TqHR@Hl!q5vY{oI0Eme7Y8Au z<hmBL{Eojk8Je0DO$R|_DC&=_zqem@cne>9-^d^`a5_OZ<1KhRxf^n}b-ym)8KA@7 zb6T%V=CzbOfjT$N#t16I&MkGi_Qkmy-zR)9r_qu^v5e)jJ2(AZ(=AIQrOKu5_bZR* z#i3Y##pHp#ySDxw(G$)D>7R~Um!qS5uk-fCyWUaUcPf){rY9vFdgw>ng%*xcsWjz9 zzWYM2d8#EjG*TkhSwe#jsP8uZK%GE5#GZ=`cF<V(viU$adBC>HR)lVMFI^*+75%A1 zw$b<Po+<8!#uWpW)!AM+!l6cwYsll*)8!1V#|5%=Ku);*mBU!5I=@Pr@R4*73P8ta zuNe}8Mm@I|le~0uY9EM?=eBa|=6vhMtZ*t~w%P&A=<QC2)afHq4d%4t(I;{;EhME0 z`qp)2EWH&ju>GFb8AB_IS~d3he7bt_Zx^LOV{onm-TUKxGG&VxCs7{icB*N->g;P< zbuiJ&T0Gm6&p)p;y2O`3G-sbks@jz#o+C<)a^Jx4yCP*GsvEL^)(6~XBUx<|sL#)7 z`T#2OmWR{VxxA#)IUk%fCKo{Gp+Qulw3ggj=)c|kek3O~>+r|-HRWTwFAhm_tBYxL zW`PfO;|T%K#NSopg<bUojJ(YvA(qt$=o45L<U85%4TRY65S{(1)YmZjltwBwy8H(F zfW^=7VWtD@x=KOA3XN^q?dtuEk_jdu)tQm@1)%rh#T?q-Yz0MURjK(ACT(7-pEmG> z=7+0r=}L~xGv#!*%$51q2E_>&e_@S!MLVtOFVC30#98_TH7=(TCgA&lQ^T_I!HuT8 zVrdck9_i}?Mn0A9v`wT4?D$wvZ<2fxkTNv<x*pT?q#p?Z%3-@-Ff$#YJ>L(-Ba_aF z?*)&EYQ%OlHb|ai+jcl0V<kWtT1<Op(kppj$+Q8WX5l2Nqx!1xw$}=>de$Zk+^NSS zhk3h(>~NP(8xTz!aH-Q#&A=Kj=B9S1wtsgMryZ?x8Jh>RK|wI=({k?}tj+H>3LqbL z;yO1!CM{3+*1RfTBI&^+s2cwe>sNw&(r3@ZFm_2+hyl>Xl31TiS5cajYr9*^`fK8^ z^xI@|gLmDH)3s^4d@{rRVuGXNW=xCN1K-O?7h{2L6uu-?t+LV45p@+><ANceR<)!s zUG2DS;9O#=_fZm)pC6Pu-v0f|$Yc5_B_yk%>dyvSR!5y5HdSfWOC=~*cp>TV4_h8W zg?<J7G|xRsk9C>>Gz2ZEoY7Q(CPDI0E-6ScAkoQ$30F-{`n^cpMD&e|zt$kagp&01 z(jjv{pxw@@JgR`IkILfkeHiYHHE1;;*$fWV21agFP+j+#VpXxNPHC1e^5z|{>?j3H ze5l&uQkT0a4Qg2OrR}Qr&=lO&Ye2_o{9bYX*wspn(ub)=xo=@+T7*J}oP+vhd7PUh z==r?mCJwJ<qm*D5|2{PQxU99#u&qCJ7u~?;z~4W%Bg)letT*IgvNP&$L|20sT5`Ng z2*B5HDd<wG9R&N(xDWlIz~1`=G8VG6s@!K@dG5|qL1Cnd1w6yibL0E55qTmtPNh#B z3Jhn9HH%{pBVhenswM0JTkw+|X-~)1T((y3faTmu$4d#A?nIxGXsUwFiWemcyO<<m z$JApfgXPop;;}2H8!vgoa-NW3ev7rUj>RKEi$iS&Eov{}PI<-mPc+0aM85i81ha^Z zJ*kjdihFci0@x00`Pz$mJ4K+Wqra@Mv2?xDM&OIna_Wh}hKh;I2NA48(Irj|ypPxp z-(I*W#@M+`IaEY#REYxe>k*xzM`;v$R`G~1_UDukznz-fI3S#`w=FeRERe3ogp$5x zVy`p57_uDO$UgEhtKCHD$tO#iH|YCqg@!(a6{72BT(OI}K-KW)H{_ReW4iu0t*p;T zA+6cyF*dUljX^Y_wHN<mp07yk_+Eus6K4`2TXeOeBc_obQo&_!X$G40O@s_Crxq33 z_8M(wS84XxXytM<p~$%R!4*#^do8|=(v;OsV)Qu7;*>`X76w=l38N<lDYdrN`H57H z)?BKoMhYeX`1C##vBor63}mxKbm%JGraW!s2It|sy|x{xrkO4;roO<-EA?*}g_W$b zh-qoGLzHx^KiUQq)sdBE%}o>*3Kg6==2iq#UlvxadiF#kFn4u+(&Exf0SOM*j4OOn z?aO{g&^o6SputPC-RZ2gF7d%x@8*Q&jP^irv`K?Y7n;<0o!7B@<6AlOQ`mXy`-0es zv%c3==1J0?xzz4(C5(>Pz&M?QZ>OM>t-r&z^wFDH$Nn0u!F`d~m7=GaOAi-)(=#W_ z*yYD)THu0csoY|gMdOE^vr@64%dTSs{sg2XhWG(xxA27q9QRbYUT(0>y|n<EuVBJt zx29q>s>{yG{K8x$o0w1Ji#E`j9vKzQ%p0#wys4JnqFE59296e{`^RQ1L78NvJ?z8~ z<~M=DYMIqg9tUWhVS=iEIz)!92O`>~^myhMSr2@@J3R2A(PF%#rb(in!IN&Kr{h6o zjw=o|A5-VbZU87wB4~LbS|6{5of&u6b^ypf@&^=2dXg}~CqUCj%ig5DZXGyJNA^Y| zxsrLL5&7uB`sqaWjib@Bcq_Hn+r9I>O6P73Cn-KWnfwWdhp*S;ZFWWB-3Rfvu{VAc z+|f=WErXTT6OW7MCL2P^hFp3e6^YE_SLO%D{6=-ys*0UV76>ga<rbG!dUy>+HzYI* z8CPpiIgRJho=*>m%)wR>Yx89L_AATx^qi(oYVEJfSnL*W5Y#uySFMWLHRHkoxPd0B z3&U1c_KLd5Ybj~n>&CcQm6zSmc=OF9U;BD2aQa>^1yZ({?DEgvV|gaCNRbemQ*Xkd zURYW0<~fDt+|+|T4zxkci5a@Q@g?@7?QC@ebrtgLq5d<T1P900{q@`B<h7g1;afE# z8_I!K`x#;>OKT~>z90OX6eBPWA_C2v;GuU+YGK7B2FC<xQ<{C60(S=VuG`=$1FpW; z8r9Eu`@{7$Qk8^(sv=P@Fz9`&|B|GD0pa`UD^uIy0L~J%%=F5nie`fv?Tt-1tuaU( ztbZGO<ii@sr|+xoGHv9k*mp7w-&6T4Qupz;h-as6$L+}To-dNkTx4rcW#op!i<c$R z#`TU10-vLg?b2ZccNTGus?t|mo5rnH&~gqyLsCP!;&W~L2SBzAJCtMI2&K$;yjiP- zC3}Z7na$Rx$&dEa)p*9H1ry@X7>lbe8{)g#iobbpAE`b`{GEuzhkh3|6@)@A%w_JH zrjep6cQK;4ccxw1?C>?Z@GZ%@hh<F910xO*B+!6{uHI7t*g4P4Ugjg^s85oN$2atl zwT(Uw!G)>gyhRjW<sC71;RT9lD87v)Nz`YYs%mc8t0NDMjSYFOZ)d(P;e+2olYUYs zmGh7bY7ERTlD!*~<@S5PrQ9cz?$Jl;L~JS}=7@iPsB0t?In!tS+D6(TR|H9?eEtz$ ziWxB5x|F>1i0dU&ehavr@@q^Ys9Z7d>(8U36|x?3@~;GljL*@VHPk@{eNS=`c?n-J zgmxNt%4&ldv#h9PCU8xJ-$Ct2w^ztpXsGY--Pm&xI_GnYE3f1wpsQZ`{W+BWV*tuv zTu<y2s4`Y<joA-Xv6%Pm`P+^_o3L4KdV1*Fi3Uf%evNWGSeylrKx5YT;tV?~c<XNg z-@8X-x{=cnXJeE&|9vV>)15Xv%ZO|Zcj)6hA3hW|npy4RiAyQ~sZ36B^H=G<>#{y) zf3sA!7}zK5?yu9rty&p)K3GZ<hTZ8r_+~1%=hru_m$Qg%yy)>=g6n_x>TMJ=0_q`N zsRd}`FQ1N&o5XX@O=i)h3g(At@f*-$G<7NBEk!I)is+L&+X_67Lv8^`CTri~RrneP z0IZz{EZmooH>^_&=5aWn*u547YTIL&HYQi#5gJj-OyF7tpgNFfg-7riCu5w!jEh1* zC=2{)2ojleANsovp@HxFndSr33kZOKx&Q(|kCyJ&2b>q`>fKlG(RpCQ?08k%>Boco zT`Myh<ES6t6`6AceV@+r67Tb91ZVgM=obw08k6hN8I?+Awf=y76=UAQc#QyswVgX{ zvC#F)O_!=%BA?`bJT7VsJ<5oUE_+oxvV`*QaOVzO-D*kydT%eixt`_V*ng<A_u2Tq zsR?VxRxXENAj^4A<L2XTna+}HFYBCQmN7g{W;$~k#6}Y`8HVdiSqHT_P$QXQQ$vPJ zqm%B3#P`r|`w~1o&d(m(L#=Hhw&n1`M$o*7T9ysDl!6H0m;sQvY_hWW#x0<q+L^|F z<{^G@(2<cK_J<wd19W3s_)1CG*M=vutQNc4)tiCB_KpEihwZKMl%W_UB}racpmyLN znToT}qQEhXnXNwGa3wUqZ~_kjCqSOTInJ-M9srb@Ha@Yx93sk5`u1UHys`SnY=!9I z%34u43g`>=w0SBU1+;qr*O5|r4Po`iN09N6dfHIOByXhA&8h~kMMc?}!=Ztr<TW2< z{@bQ>3lOsm=`dOp%*RI}A7~*t1_U8|W$ZMdT=p9H0nq3J`<D5P4FIOL?gFs;aWMM{ zj+vTp$B<(mz)1n<9B4lltH_Def2}k(%8DiHHtwPrMNr?((h05}Q-JyVQ{w-~REWlp z*P;MQd4YTDy^)wOHr1bizVJPzAC7C*)9cC6<21dWUAt>c7zmo)iIOsej8N(jfz3vQ zDOa;raDK8$DPH^0vD~^Ci4eY-@Lur@b&!59L7o{649fIoKyuNH=BHCjHY%?n)iI?) zO^Ut>@!6eLTMy|dabIj{L#pE5=w+wEDGd{`x8J8<0AW<ydQ#Gz7!0tfy8B0`XCotV zE95{=y@I%Ni#Y)D1)waNLVt4BGr#9KB0`VK<_qahwmI(5t0>28zoYX#Q(10eXR~o` z$9J9h4t5c}Iy5wdmQUMwC!Ixgu*4BoQkNjCX5#tUCP{>!bIwPn5NoZw{^al%^>=vi z=aF?O10~~MkP|wv1`V^ehI%B?1I0B^*o{-OTSn=l@>%(uKJ@8Qk#$p-o7|YhjA9Nk zI`7bL5-l6HCGT6*+H2iW4`mvYmB`%XEjpJ8iWED9&j*R(;nk~>2}AluqMB<1wC8+_ zmV<QG2Xg}DanjE><{Gv6+&td;UC|@siMqQx_+FHpoyCDi+Z?)dYR!v=koGsBjVz>a zA0aNty`Du#iQmYVtT~UB-O7qic>|+F0EM*L4PF<s)qdBtfIxIO(eGR@mpyEli7)2~ zOB*NvP|??%eI9^n%L5gBkWsjRxB&APD|4;qaAT#RrDYyYGYZAxu{A7KkM&c)1-uRj zM1tZ;)&AE}bnbqNQP_gvJI3PR=$G>DR%jcbM6le{g2_ap3}@F$o!6yV6wtK>bgkuT zgc`{<NMis}cK3{7WLCQXI8RKg$xb&=Uc371Ne%!mKF4PO6xV1>(n`zT)?xYK8*k4h zUl%u!J~O<BA6I?DM&R~@ZkpAa@SAPKTEkcI$kRHW6eSFgp1Eo`4SfAGe+A7Ktr>ak z^W+sAfDSA6J@_2rxTR^7m;ib}7$Yz0PRS&ZNB>o$V({qe{zRh?^vHP-o`v8c&#bf# z?2LaI_C!%en!wt*2S4rA_Q&MrFhC%Hs-h|HSmc)nb(9kzY!a~n#ac&VA#^YL)FkVj z4dU%+$j7&;%f{i6o_cyf!zEZra?xEr?_q8jgZRk!NW$6`1^M>0av|t<Qz^f|K)$d~ zafj#wzV<z1HgJ;-5d#%}pTmZX?*GHyTfa5^#c#ma0f<G2gdim?(jh9+-7pxT<mhe? zgOo05r8i0jBStD9j1dAGlWvd}kd8eEKi}{7x}HDbxvmF)l8bHUeRj@yy<Ye0zV9=% zsXUh=y}#VdK|=E9&AN0grNC3&(SxIfJ7x4?<<RSJTkHPjFVf`D*W|%WGTYX!&<`zi znv=Ft*fzK)g+k_6k&_b&lCtNif254E<zS|^_BxlU@Y5ssOwi*3NL@ddCikr9>Cr~> zpy0{DcHxRIsPC~@EQlxmc*u=5KpJ0Z@p9<Y_P<@g5dkz2LLAH??*OL=hJv{VZ&Bjp zj4)U@{m%-M{_snpinmj*E<dPMcn^-5G8Mf0j@XX%>0=4iw1Sl#2z_BfZp(s_;j58U z->sZqktx=I<O<H7_S(wS?<g*FU*>x#@@P6GZ7&8<PqPz100Hf4j)<5T3XkBflR!Qz zMn~cF%I=vh3-`OSvzy+6v>u5VqqSsKoXHdE+!lMGJ7UMEdAa-5Sp+=VGh0jdg3Qa) z1o(LMZoCnhbN0Z6Yl^(;U7#fi>h(;u)(wpW3=3A^R=mD^a{J$MkPA;JNrKq}#72J? z)eErkd&P}C;ndVGMpT6M>>}S=ul{1Bu_O4|`(hnM+6PDC{uHX4SrU-XJl}(p;lRIX z*#3pm+tnYTgn*TL-^@G7tbZ4+6ohK@667)`<b=LUul#PHw<RKChSnoOxr2kt9~uNv z(ezd|eW$4n%SWkH2eP_|XCmUHg*LOZvSvirf&Au>vaITQiP9njQm0KVbKRdiNj3Pe z606z*S`ZW~fYpLD{q6I^hM<1k=K^GR;yYs>jC%05|B>Co>5VeG5~e>(3Th{D_gJ#U zwk4g=c}AVehfLuceS84jx<_~IUgy<El!cNLow}dWLYzke?wZ$+cU=>i-4s>TUW>Pr z*O_N~^*FY#Pkl2GG-p6B<pko9>kmjF8K1)D!#063sM4hTe!z@!VA2=cbIEH-(uz>0 zrYlyV+!RBmu<);-ErRM^+-6ans~<&X0ZC24rO*Av$eixT=d2aZImg}Ubq6+r@mybl z=ztqr<(6Q_Dg;gbxx>QvY2g)Th$jYhp#wGhK%lbi(>~z)h{hIO-HIG5{KKb`vRKE# z<hrG|=JiSv#dn-Mk^}W{n5T^?ra!T-F<<e42In@qPN|=+CL(;hxCW+M2a&z#rU&?$ z<Pr2^vWC&a6~NV*NsxRM@D(gCpHSa`0MK-0N)P+>>FO`W8&^Dh-=j__$n3o*-ng7p z$Xb^O+rcEBxW4dN^SQhbDa}vy>qKIf=jgV!<h*Ximg_rTBT$(jAF>=?U}9zZx?arH zFXX^3qt-I$e2Fcv<EkSxN3OHdVI(GAq(`!^{d{~dy=~B#j)_&S;y|X_qPJwoTz$z1 zOqJ~TX{txEQRhV^!gq$?4k3vHW+bI-zq9D--Vx+QUDh4Tf{Ru)Dq1UhcKEyeGG5eC zOhS$0Maqstm!IQPjAhC|QZW^5<%kOW?y?i#E@8+7IJEm0qlZX!Q6{DB&0qW(Y(V<{ zbnGyjPBJn)8+lu^zVNUml$;=D>Pw$A(v}$cx~R2ndUF1>IyKUD0l-{#fARMc4EFf2 z$W8C#?M3yvUyTqFi)$2p)VCzD+x~*I;o(o2xQ=b8e-kBl9vMD(D7ZYEQ-TQokmnh9 z=gBRgRH@_FjC?itJK(LLswGtsBe_6mG7#_>%V2ii)>(IBi7ozJ2I(0sU-ErcC~Z`M z_D3<^(opm6yKkRu!%OKYwle0tC8^};$Swo+`#6>v0Pt}qm899%#{b?=9=e5A8&!uL zY-|MITPB*|@{3^-Te>9M8Ag3%LsS9|?j1T_Gh$^niEY*q$6iIE(MKmu4Km?)_mjb} zb~a1H2-P+{0cu!t*U-$a=KbP&jrPBS0RpS$WfiNBz}Y|BB0QG8u~wz9?Wrx<YWn`V zeRBKT_X(JnV?L2NSFbG#E-QQ4E59*u4Bu#B%s{d&@7;7OkfOKktNPyZ)N>_b3O<g6 z)nHBQJbk@&uftDOWsGyg!>;`ux;+C6o0~NlwA&Neh;09^lKjW~-1Zez0-t5tQ9|!w z-S3fWwORRgeWPvXNZ!<MX4f}=A98ZIvM&7Rfd9Colp28`pT#tluQ*J~W#bU^b{Z)| zo09pTav=f_AthQKGuW(!;`JemzcFp7bvM5W4|1`J=-Yx?j7~31Z0{a9?VZm(ygS!r zSwWZJFK~<K(6nrG!^c2hyTWnipi80NW8mUJ_|l)YxLGVulOKz8=bTS;l6Fd4O9D!B zEAft@!guw)!?^9C9YM(v1WI}4^|<Rua%%ck5+E=yMA+AbRKcVPi?mX=u53--ntPC# zwoq8|Y^8q&JHf<*C|tKNWGmE+;2`XLlz8v|p<?%hOkc=-yks;$i|fzny7Xhi0o*wo zH#IHq_3W*Ps*G?-?@@s(Jw_DH*tz&p)qVOWbB<i~fEg+MmHtj-u5j1YTQdS8E*_L~ zx_)U7hu_1T77ZfjA7xgIMC+Nsf39nvI{of)a<1Ojifk}YUoIusp`HwC7HC)1TAIF! zSp0xg;ccce8L?B(MR2BK7wG<0a&N8~M(21EgSU@+jUGbW^9BVypamI{7UWQGfhrG` z6~LGsNhn#or&Juyq-zT#nAgh!0!1wfKHHhcj4UVw`m0f0+wsUB;k%bzdp$=*E2!b; z2qKh%)Y~fJNWXHAJ@dG`aOicpL*`sKDjIv9<c+)=pz8I1*y1MpwVr#!h0f`_AoS|* zzaJ1K6qGaRHACblF+aC_KKFas;KsXV(+nR@258Hp=E6QLF4+J*WBoV~Vvx&Z)EWe< z+&d^s_33qGy-rXtUaYdbR{BoBl_zkA>uMV4(JLJ}i`t`O8%1-h7iTE92n79umI89c zPWfAIVK9EcWxQ~}e`l-Kl3vJ!n+oL2qibk2t!Ta4rk9qPB~NIhUK(x@3jRn^Dw=10 zG*EV&feX3z7U`Y0)ZMu|wdBqz@FD<r_i?ym4VU~6C`H9GCFWw>1&)GmW`DL9`+zXf z%21(7*UsLO1oPwu!<OLb;o)*u&rtU7o6RZaPwdhngW{h*a45=Ix?Tln$$;1l_<liI z)~OB;Nt1${1SBl#8sjNn_ipv8u4F1y*NlXeO2g-!%J7>{)j<}ZDuo`?9YpvZat&<t z&d_!vs}%|-)$u=mx;Y^k(86_qtgf0rM1Iq&h*3Jw+c9^9C$P~ZQq_2AmKavRbsiB| zi^X4tsTK*L)R#WuCl=;@H|#KQcEC#eeIFn7G^G1M_SMF}B9H5f11V)U4-Wwc6vqfK zil(-^>UEbPgI&CiTr+o6v-G%*wM}MHR(3{o=`Ah+(x7$8QRb7<U(@BcvId$aH)NxJ z`eK>roBu?+J2=Vvmz6_93wJdYsAx*>0%2a{-~Ofqgh|+^?U9OrZB#WU2COY7A&Y!F z5E5S9QFmjxV`^f%=2pQyIx_Xn?i?kBVJ@^?5vuS*sCWQfYHje$Aqd8m?em%8f4Km0 zD;UjyQH;l{ttde|^@7>VH`s0<v;RZ%AD1|on>q~kf4mDCFs;`jvoaRnZ%b0M1CboA zx!6veQkIDbx`mdKKhR%(56f%LNsn1oJ|^l0j&bz4?QFOvIj=PTYx!Ooew^83RrU@L zMYcaLzkLm~F_M5l;*_!vIs~g?5zHQaTeVqk$v!D87*?c7lUTZCs?Uf(`;6$260*3Z zXl}3|RU2Z(lcLO{I(H{5^_6Q4Kmh!<aa}-Nxd40gs2JfB58mn|EM>Rcb*hECb(#8n z_qIOY*s(sBj|b$%2kk4^?o6!w-n8>?giskaPFBHR5D>Fg^j!NX*Gw%f`**FfVv_rj zItJ-?)wy^`LwwfWa$m5jcadbkvz}Yrt~0DZ5g|?GD<e-}dF;eoCP<-pMgJWeHV-9< zy`7WP0#u?v6N9z%Mi!?6QOkL(!_-D|DBa*@Mhw^FJSD4Q&H~^7Yo{m;3MbBg^W8=? zs@A62t+no){?I!j{6Qd+s(=RkVLj^*NHeRu?<DwG4-Nfs9j$tdL!Vnv9ULUrXbzec zV?`ar2UR)FpqJ{=*Oyz$hUwrApNHS0BP<=POO|-FN*@mSr{wpd)qO<Q&V$aEc(nc6 z#jY*0wYB0#z3m0A`pgBfBd6i5_pO7Hd5a4$YZzBOD6d^C2U3%ujS+teo-uTem}{>z z70QpSKV?p*CsaP`JcX*uZJ~KJAC@>oe|%;jF)*|>yQE;}@PSK4D&_^`XlXg5-KAtG ztjFr%2&0cprP)P2*2)Caqp8)U?>>&1WpV}+=_T?yM*RAFpZir+v+4}zaG~e7hc)P_ zG-3=SxHXJ=@l>t8Hal(I$MN_{|B{<TmdI@Dp9LN@fKxS=FG1m0_kxd!f?wb7d)W=N ze?bxyw#mc1*)i?RAuL!f=&V2xr<<Y;jF;L6$<wvJM<+QPpathzml#yF>nmGhKrf!r zxA=;7%(YG@R0A4JrEe<v-k*^_B8M9bz)&CK6|rm@Nb&fww`7-my=b&-T8UmU#)5gU z>(G9o%ZpH1Hs)hy1%LEV+x}@bwmIE8-z1EKa+z&Qr7F(=us?%eUN5>}a>|4YY|ge! zJ=Dm{vd$%~xlOS<w3$}WO1EqYTYSyKB9Pm@6X7)>d;6fWHR0;A(YDny0>rS2?lA@R zZHFB)s@fl`-M5*`UZEhEyzP%aF<+R6y<t6J)(-|$q~aEkEJpGru*qU)Asbjmt8eO1 zu#Q;wbX}b{Z>ex4&UKW-jjxO78bZt}E~vB@$C<OOgq)fWj(ARZ1f}H5>WFL3c8@sl z*ur!vdtR97CS8$Y73q<P1iIDDL8{jV0?#ep(M`}GF!@5?PaWT&mZ;=)?ii9{YK2`g z^9rwGnj;;cCl~F;uqbuZ>w6RTidm`e<3_qi0(@bXcV&@DmC^LYQmp+_o{iV$FkY2{ zA!A5@cJ|`$;;Nqv4cH~i5CROVc@`X48y@C)7y@PW2C{mE#lXmeFfM69EMtGTgO(J^ z(S$&&9y$cnYRhQk-1ad^A+t~G{`=VaSDx*%kKObOUB$*bsZi0iVa_?VFVztzfs-n9 zbzh48e*P{fczUZozWvZuck1gKRGcR18xAH|o2K65x`u|4_@iy?DsRM{gbgwvfmg56 zYP|G`!X4zn=HoujT!3sm0*6jMnk(h)qbaV|c*i21GSvIZL9~bQ*o|BoF6B)ytXq)A z+}SP>yD1xYpuo5lA9%HI5t+;3nz<eR1r=MOZwjsAwSPs=cGV}p!gDc;bkC_s)tCR> z&4rEmCrPRCr+qI0ciJxFS%83`DXBxibXK0tUFkp4x@pGS$uD&&@2#Pk2?kmY%(Xvw zPbQOVk})kd#V8md`kqg+ct(%asdv0M5-!VelFNRIi=6!>{#Fn4QKpqWeD%Dq*-{t9 zUV({Ada(wO5V7}8)!vFJ6zW|egsd$X;U*^W!D9MMUP)COB?#-f=Qga>?Ipgc#$_sh z8POghpM~Rk>}=EcK(k~OtIBq#5B3Xa4i?1rBP}C)4S0ecvJIVhI{Zc?)sOc|d)n{H z_9*%-U!44N2+E{@FORZHx#|lGaGr#|QOgqq>FS4{hY=8gq4i0kg$~GyRL1rSn^B7e zvR94iZ3(R2zU`&Taot`kupB@|?Y-x!t3`Z%qKn!7>#juI>zki-WfbtSJjX8(WrD3) zW#YaLk;{Z!DvA0y{IYnk61RZWz5!L|vI;U<n-=LQ(Vnmd6|UE8?>nQ)VHN*39XlD^ z;)*VGA_1877eJs-eBZtu<6YN{y5fO;mR9gezl^+{mKG@F7=O&^l~K>MEoOG-|7oK= z+}fBy<E)z{`|MIyZ&D~gLZ2$$?%zW_y&L{6>#6NCzblp78w4TATzi2HNO|r|pRbT= zRaQV+L7DZ^$obrpU&s5wMf9iLNXOq|T$+DM9vP>{!>);7>!inxWfF#TD5)oGjCG^g z-t)=das57Wgk!qr;FwhaZ%(SztBHO2RgX$NnnjWMslt%^7<=?=tdMl6tY*f?{lgk- z_L*e!8=>LfN<^cjF8D83Z`D=~cPIHW(@Sp99^Dt*cFuwBlndZ&`gKxQgHu^I*z~uM zVcjl$2m2D&UNvg(pd5V`T^%?AH`3~toW>-3P={U~ltXe+y7h<=-_%KM$D2$jlqa3` zoL}URp9#h&XbP2Ak=((|GH2%qdqlD@$h1R8Ca`smaVT3#r23j(zN6OYHJoqfP>JMx z$#<vW()72no&~a&UiiV5b>BhW;^x+&oP?>3Yc}<W@n$0Cx2w?i5j9_5dJygLaYM8h ziPdE{Q3mg7Y=ex}gZ_y(K~vH5J!}#pN_PYDQS2+HPcNdHl<;@B%l~WA<MxsD=k=(S z9$sHFlOomUonPOqge~_~>6Zf<{r*~kXTuAt<mO#WKB-aL_&Ji6lY^C&Nw{)LKG0n0 zCLRBD&n3qK`Jw9#BH6|g0ugWerny?3U?Qm6_CcNqtC|jMrjX{^^n1+llj8$lq7Zul ztY(x43wXsuzp+VeJILbem6x%Tf15(On4gaEs4X1bkD~He%yGbZ)>CHF5t7r|*5VB( ztKJj89YF@~zmz~$E&A|I8cEL=3^CV1>^;K$#r)oy=FD#PI4tD%Ehif7Cb>7vqr6Uu zX|DRCmW`EV`!ybUL<RWi!S-TcGj(=T{dWXTP^Ot_v;6A^9{`|Nf-X`fnFc1WzIxFT zBxF~Ce^Sf6yRwggbwjAs6cCQR%i(AnkjuNgJ__Iu@68|-#78gn0!c<nSbgm&iW>`5 zM+bY_x-5jK<sDx@EgoFr4+9~!tnJLPNp$?^$PdUz1`|7}%A#mO%sw;5XpaDPr<Cf= zb8oox@5<JU>H^*-a7~$!K4$#co^&Sx@BIsKbP2$<<{U}JI?zV0twn`O8K(n~q+)E? zS(HV{d9SV3DD;a6$v&^^&iT#e5v(B4rvVzU2nCXRRyRGpsZ~!GFE*K$4;Yh$4}tHK zbxYw#=gc-c+(O4da6OADr}Y9AiN6t}cZES%X9v*x82oqu<ZzGJ#J&hC-adHra*X-b z@baI-s{s6FJ0n`xTA%sEydF5`RgU0%{8!F`l%#18h5`l?vYPJVnZQHj0_4SA+16?P zJ~!IM^j*NB-DFf(SI+~?)!N%Zn>-p|C}U{A30<TB$eO{yb&{sxe~Qjuz`0v$jw1pX z-uU`<AwPZ{qBkffFgNY&?*8xvFPc*Nf4NxgQz(L8{`d37=>NB|0yFXdB?qv7s~U`X z%6HBO2x9=8wBHOs&0`?bjh?L?Tq+@~#Kgp?6(+ERA5?;1C8!EE3(y;C-u=f>2l^|o z(>_2%-#jy~Xn)0Y>QQGLLHs8@_>UYP*c?PI4Qy&O?*f%^4hLX4gI?`FYXkt5A`tyH z?&Q+jDWNT|+?KMT=1f(mtJg`t98_{~zhCGp`+9T0?0^56IJ|q*cdp0(pUlBo=uEJn zb833<7d>p>W(xFs`;j}=pvl$T$&@7;jn6b~?GsLqI?&<$AqtdR76Ntg+4MKAYywJc zn<<j}PLLSg4gIYrK|WWxg;P$4DXCtpoPAuHbp7`iHfLjCB4_)be+ziwPg`_9J56CX z{(bhpwNCix<anT?E6ZHPX4U)__G0sgD)q({?rBTIi7s7ftIPlW<(sKwU>Wkw?B|*% z5hxn~v~W8w?#!U+TOW`R9Pjt9>?p)a03oGs<3tRoz>oKVNV<m+Ae*2yx8`}}-M8_N zPq8!OYo3oTc$IKO0Dl$7nj@g~t#e0t7RI(8IWlJsF6rnjJJ$FqWu9Qajc8N>`1$0X zyRT}E$y;Na`~PjsprIhe^QxffuI*&=1t2=UGY^An^+7c*t#fjC$a(oBGDkx?!+X$% zj-hRFanZMNmwE#<7b0F7CNEznh(H^hST***5fTKR%W>R{zc&2_%h{sY0?qSO6{||a zc$Pdl?e4Mj&ViPOh}=19`Vh7UQ5(SN>ia!*7Q@Ir-$tu>DUzCE1aP?u>MywMWm`Xv z@H+Fsn&JiQXkcy7sV?#F^&<_9qS1UVi2yH=u`x`JTix%y&09}T%{c+Y14uVFQ94H( z-V*T`2SM8LnV)mcqW*q<cZfVDUvj!H@T~6niA71M4hpI&-Z^{nmO5}&IR~B@LCpdv z=Je==a@SZ-O|-KcAmZ&G56nmG1vWSL6FhO7EBB$<deG}{JTKdnA~<0;AwUKfL_A_# z0&nx?D47KVme9}iL9du8syC59qG6b<t4rDOn9-~?6NBVd9Yk^`uzc~pK;rD#IvxA+ z?6b#}<KLG%pmq(u@ob)lds9F4f2rk2i46A*Ysu{-Y|o0`AREvl{?l`3N>y>GKZ?j1 z8Obl85L*6dsoS(c7f%}hD-VJAZF-v9N!bdcCWYzTdhf|vDa#6b8AwrLyU7;fGBLv= z$T<NU)W0w2FftYS_EzB2X`brhi^8SjF&Yi+2USI<QAl;+0BiN2DY<aZTyEw5s>2m% zuIIwh1~*D1YzWHzZ=O63e91yvnflz;o&zE!0Q=WcmOB8Scu4NJlKY9sEl@HFN#5+( z1gf>5AV~)Hgzy8~oE#406#neKK|$2hvdC*6O-)Vf$9Yfn<5?C0ag2GkzFm(TgzvA# z19%xdO-<`+>2g;#3t2O4za{DA@Tp)4hC=j6sBa3Bh6tB558KZ<z7CDW)?_8=g)eN^ zjTP?RkMhC48jW3k1dH$#7$N4@3Zo#UfKo5?V2*Dh0h3p~w7r-~ylJ(mM=T#BPCGkO zZj3up8#F*sM#ol-CReC7B>PY&fE2ohkXrjO?ff&a<U^dXEk4or?y!czS$RTb#P|H< zYO?3T3<4Ntm_3|zg35kJu*=B)2mq?loNY6Q1lh%JCDedJPI@VPv`BCEr8eV0B1Wbt zy@&@f!kM;al6(JUJulViXYk3cejq(3S)W?!7K%RC97>Z7wr$p)2r5_;m=sZhI_mR0 zam2rQk`TyuSrOV?^R$uh<59aF;6_V76U*)^opnFe7LflNk@F{;;#(=z@n9h&QR+GA zwzsgReA<iaA@{~$)bb*sBhVy{Ip0e{9Db4vQb$1QR{vrDKKWatH4DQ9^??KbG6x_U zw0i=vYduj-xVKdjU~Nk7h}S!pBH1Su8q3FA<Z@TKRCPDcp<d|(SaYb$z0}NeFdDtl z=3>7~dS58u_BS3$vmlwDXu4BRxl$J<2|^mJzR@)NjjGelQ_sE;o2fp5$<{Lk*LsZi zGMwJ<Ek0;H5C$<!1{>5h>fwsphG#wbq`9u{$c@OHKNMBAuRV2jK_k15x_WW@;8lf< zZ$3yw=wAmUfedQRdfA`_xV;=Z5g$<kO9&uhKCOTlzi_~8k_~qfv(yT+W*~sUM9_}h za~E&?OPFf+I4GV5&DckwG*NO{C&0yG$sKXpK4~-cVoQ>M1Y)<#!U4#D?a-f>-v#S6 z%y(s^Y#$yXnsm*sGhVF*2~t)p5G{ciCy}kJzf2|htrUT%QodTZW0!=kUh)eF$iHzO zreRo`1XOpL65Hfy6Axhbqxx-LHl00l3wa&U$q_EI{qIa+QA8kUqvW<W-hFF1RBAFP zI@3i)<baOwfsSeNJm?o6cU90nQyO#!<~9Z|43m#ePG@Ij{Hg$wy!F$CjxPns>xvN& z@V}e?Nt}edUY7l2etQ4G(>T@(G@vwfedFC|Qf&H~7prxefc~A6ro>bz2-r>}U^jv6 z?(E<J?P5D?cc!19C3FvHl`JhZ$+mpi6b!K;blIF(E&2&?!ZIt=jXPT2(}?skOooLN z*aCa8T#m#2eW3RyBeE#wG+EG#;_}=31_T3qeBzW_!(Cai%V3tzI8p198CzS&$K0p6 z9M^p)b84*&K{o5#nA|!EEHuD_Jv{UB0DNl@wsdvFiwi8WWSb{bEbRqpF-R~}sW^t9 z!82#3FN!W<z<8{v_?T`Sm;}UhAnl9t^%H|ZU%Yx?nf^JC)v~SMUD+>9S`U;y8!M_O zavfv=f5EMr%co%B5Ks4{c^;J{`GQaUGvQgE<Oi%7fP9Yy$Tf9meR6iF_G)*0Pp-U1 z1IZ;|sld95gmlIljIqg0jg8^tg@f>h4tSDy-V$(w1C4j6F}3&2nU&I+YOW?f{p5Vw zVa|vba=-DTMK;zJq1t)_o8H`MDAVRZ3_w3M{qkI$7SRzS_um7xx3rX$t}biZQ=inz z;g1`3XEM>2p4yFYTS%u8)D)OO7R8hgdEl$^`El3IEla~&9Tw8`hh&B4Hc%e1#>GhQ zUL}nbElqv4k2wk_?1W|BEN!iFNLKve5s)LRIw!N*f72xa0SXe2m(-QFhnjjN5KDV& z29x(Hk{^4Z!;`--6-Rv#6oRk<iO@8L@>e4Kp*ze4ucclVUTBh*tQ0foZSr>HQ<0@E z>Ck0f1xqTYosJReEBMRO5zO1QBMA30Z4cC37G`$2@9-pWAE}>ZHJpTi>;Ub8)iONl zO|^yN)UCdOf=gO`dQj8=ejS)B>~PMp!8m==p1P4w^8w}CyB(2|(&EEe(A`2urIlJb z?4fr3B`Lr5lYhsz*%teRy3hX0f%Ip3fp3X^t<s<l{cQBC0sWdl%El9T<UZV~$!_~H zFl9U`WMRJu<VTHK2!<KJxtrEUJWVa;!i3qL+sek}Uz)EtF3fBC!RyItvV=M%0n+fg zw05R;JVY2r`-MuFXT@?;FDU5C2M)X|Z<BJ+#7X3}M1e<DLMtUnkaWU}&H5@qRlIr6 zkZLxBuj*sutbqfiEa~pSfh`YlvGEGLbe?tLV$tCDs+rOPL<ttQIYb-0!hIhGz43?w z#%z!-P--6*Zf7+^LZLQ9`d9{wzo)^4L*YNi?g~dRqZp4s_8WU<N=)L+weD~D!Xp-E z+^YcP;hzgOj*1EwQg3^`4bC_8EAwPapDdu{25hV+e{ydWre$+zq`Vim{Aeah*A*8A zs@kgs;$7usibVzS#Kg8ByI`pyrvnCHcEV$|<_VzmR~62eUV&$IGv&SPZ34M?xOzkw z=+bVOM|vzXN==^oelOn^7S&YMy8_N1R0aN2*P`MuMfqiQm5ecSARE=q1fu>?mrmhe z9LnHy^?n(ezT#D$fskfKsR3sX#OT5XE{EHe&%*nut<Q&S<_o8yDAnkk&dRAj^zTD$ zK?MzRwU99%XRI_;L3G-srrTv}#>cY$*2?s{B?u+|1^MomE{*i%0Y86#Y%BgPK*g%2 zD<yM)NsWHo)TjTUY;fuM%Tn>8J5NibsU{~xXRqP1@T?&tsMwG~Z|oG+hqWX4m%$;! z&yGcxRb?E>sjqJzpPn3zMtCg}gI;udU?w%-qfiB4|Hv!vFI2cxhKotC4Q?FKIj=F= zJz+|YM(|7Py!!@<W(Qlxv!V9-2L=KO{jmLy`@1~Hi*bA=P`|mh$4wXuAg8Nh{UT0O zn-iEUXcqLaWvdgI&M)ko8t^%^R-anWeM;seg(RiiP}U!%I%@8%tA<!8b3aUF!Gv35 z+g?pk!OJgA#f8^3<oyeRqtnjdAZl=!?9J>O+bXxg{k`P5cA#|sdd#eMopG;KB{-B0 z4&Y70;;AY?04yu1D5cRnxz`ZoJySFWPFVk^)I{jr58|ZkSb?26yx1M4!n}M$#<!5j zfuT#VicpRST+(IG)?m+gc=|~3wxK<*;IF)<q_5+Sc(D)GlR@z;#dyw!{+p#2pO+I~ zZ=B|i8E$-U>NA-%75*VXTDtbD=``8~Q(>2qInVMmTow5wasL-xm(mAOL2{e-E(+%@ z<)@^5m7YITQV%p|NqV30PDb#1qS^T*_xKAxzSvOBj<U^vHtaByxt6wr`sVp2v)o~I z<;d=+qi}D@LN1Lys;VBccI2_n=!e?ge$w}G_sD-?TasYLL#c*20z$*Zyg!zhxh;Lc zySm5-j^<DHJKD+3p%EoH4Cl>h9A9j!^{QCW{BRc4sd2b@vD$$Tg<2Ky<5U`4O5EW7 zJm&U;|0DRu@P7Z!lFwm0>W=gb42bN2N%ee(_*Nt(+(OBUnhJkS36nT0TB=v9R}9xx zQ&Y2`#)LP$eGu#4s+&=7X$YfD*C~W?2E}&q8v80>=)%eF%hLZ6QV?#o;DSWhrXd~| z38y|B-jt$^ir@Z09k(iHn#ig&e#@LDnd>_=(L!}y=asjuQ<FIjBFLYgrbKEoq-5?1 z!X4c%sOpiPk(FhoZ0j~6w@W&sh4DoGK1rM7p_0|Sd2v1~IkK9sl7Vdlsa%uYZIGRj zVZk0^Ts{Y_`^_D{R0KnzU)>9IEfX3}>>khG1zOUj$^}`@vFDgYmAkUCSx;CUv*#+A z$y#ME_|B?{iHnbnjy^myg!*+D<R%u~N*Xk8fAIA^=6Mt-hi_1D+#wVb8~>X2o71rA ztM6`svg?~!tna4jh^425!2M+LyjAXpBSFWsN1XBO`D#PR5}nBk+qC5b(P)YrH?}Ag z2|E^xuXFp&PS=WKS1$u_5c42q{aDp=$v2CzvV)XRouHPNm&XXlLQe+@D-V76X;}7h z)YyIGZPN6L@e68~g=?Z%^%E;Z66Cqg;XI^<$*o5{tKYDZwa)D(c!6-8?_-JrLu%l_ zq}*1L-URIE`f;_L{2wkE_E5~cw&_i*@P@?;Rx#~Fa~jNoelb2?<oj59faA7WX_9?K z*~)^GX*>+G@!2A=%(t!3A~9aXc3Cq)5=A+xg|vrAeTx_I$nCEcRDzdx%1-v`YerqM zPs0&xfc0#ks=TJ)-hwJ+XF-!Xb)(?I(bLI>SiVOl{2+d}3{`R$Pi{P!)>EV&?cHwB zadV+#-|2<Vioy=&oVn0tU?i(sA?83Sgo-#UwM|1dl*H3MROSKvPHp|?rA0`WrVaK! zW*?noOG?Q-VEkl1<Ry@5k<uNFPYzOmKer`o?Y!K@|B;mY5q^Nzp$ty7?lY0E%A7Qk zsK(Xdg1{qaBmNA?C^c&E*JwhGjall)@#w9ufF361^`1@_1xXUPQFkXYR9VeJl&;sI zY-IaUsKvG?RNui<I6dEyjxtAL=Z|r{PLnBv06g*&ED%#22Ro2me^BQ+5Syp_8=Y0z z(2y7ln}~Ge5?Z%I_$Z&O=M>J5ElBOzitX)cvgfRSUUf4Oa`NC~amrrZa!dExl5nJi z*Bdo)Yl-Z*i>uPdEcyT!lF1GBQ569MW-;sVywVBNC-D{8n5UHavXiCoFiafaqaOSL z_QOW^Yfd(zWKCzpOe4Wk-nDb@UrT^0k3V-Ke$`QLsGB^G4SdcblH}jUv*QK$StKN8 zZe!u6!^hLwTB!(Tw`)aeZZ5XNX4SJ>1A8~M11`?T8&Ri@nW%lY9hOn@XrL0aEBU3D zZz_aeYOpWt&fo6I^Cd=ht5+78axXe6?Z`k!ju(Ur?e(~KzL!I`25v2CVJev);`)tK zJeEX*(+bw`;jjOf3!qv!D4SBNFbU+1_3k>YbM!hE2eLKzv6?Fj1-~N%3+<y>h**w$ zQOr*;tTp~iTXrgrmX%6wF^EnM>&SQ`*`{G-$wSvBMK=hy^L^q(tzez*oLKem^1<VJ z37;#_L%Iib9t5y*Odf#eY#EY}mNwOA@20PsZ@rB$#^tE#Vv%eKikpv2+@aYjY#5QT zS<zlosAt?7&zifq7+0B{Wb*9ntc90}+V$rXiF|<ynQEb5UX-N@HV-_x4R_pIa!(ul zkVyALBbl#J8;hh8iq~=D95_bG)QCkVJo#+n+VGIzv0al*tB&!2rtR&BY~YK(lD7g3 z<G`?L4wXEQ6>@?S0IWCWuZSo`m-#K3Ap4FpM%g{6@Gv21<rH)P0>A7mYP_kaC8-2- z4UUQN@<~B{$=+KQ7kFcuG8)@l8f?@)gF(vjBzzlmmai>KCyp5i7TBDQ2l)D?9lK=) zi^f`vdY0-Ld6x2%om~7814`wVko!?`NlG82jj_XZwY9Ym3u!NYGvmM7s^MGCjYSYv z-0Jt=J-FKXXLVKO^b3P*T(<xU*xlIp)^J3~<@RGR;VhReK0RvXkB*Mss=SI!UY<=r z?e6X2W94B-u;^I7y>-e*Zxkk&5F1kir-wKu;41sX)so_okC{Mo+}y$37-0KWCis2n zl%B;`mP~E{9K35zeBjua&|~uU!Bw2UbZBU3d!bJXrKOkVU3j&1+QL}Wev>pqOahNF zVJBlgA?oD<nYN#pq-4fuMqXW3Mn;j5I;tqcNRj>%I@}#<T1AQqz`vR=S)3SwAEhlc z=E&uKS&(t1vknR}<;{?c1<-0G6M3tKDvm|{%iE*0BbBF;lEye=`P_bQ$z(mm3L5>h zz?N;%F{3`$q`a`naI5*z+XtX>!Z;<%_8qTs%0ld6N0=z<*Ibn<J&kZ4EX0Z0?Hh-s zlX_h)csfhL%DJtri`td@McdH7J%P4~yi#^PxXzSZ|0fGU?HAI!_f9G+pB&TyBS@m0 z(O)6)J+Xx2QGuPRI0=rUH#-5Q<0A*^vzE;dfPJo#8T;f3=ZV+dGDyBwoR>y2pbt2* z7jM#aVEF!?S;ROVn_5T7y%tFQ6BBoPUBi|1mp^8XibBXf)xs6KEeV?i-#If&jaJZ0 zR=W=bqfIh^tlfX<YDY<N8=S$hlp*Hj1g@sz{(jBPWpNS`SMKzo5y$5@`ET-q7wREE zXs>{2&|v^ihhLtq1M#cuYAe42xL#xEZPytZId(~aLBfS&@!SjWpZ|W|jK1ySF?-i} zcctx|k*$}PzR<?_IP^U70DL#g;cb?eNJ=7C&`C=3`}^L|`(&-oz*d(iB`z+Ysy$fs z;OdWAP-MH%r5E_Py9;_%zXPqQ<G{oVkvT<#6}Sfl1;wHsf1zh%b8|Kn;N*Z0a;jgz z7L{YePh*poe4ND1UB1V_z#t$3^2C->PgmEOqrhT(j(Y9;%*aU6oPR_7bu&q+o&Tey z^#q8vS&ML@2VLRh2DVQ?yXG)y;|MIkD7lAM-{R!xI+onU;or<XJw0)KXORftyo+c@ z7*uRQr{g7zfXP+La&pz<Igs}~%cB-E$`^g&FAoX=3xB!XTOBSah%O_TWwTbyWJChy zeM+*DnZdt_Y8ihu>M2~AZ=1Gpgo5w{K_~YrQ+ZSeb2!9BMZxYnRF}W)VNT<l<@NBN z50LSb_<#2oydg*0c(-_f3lcC&@WcmK7Zw_Ury54{dS!Dd9tID-SpIy=y)0-0JSIm6 zKUs&1R_o=@4;KEr0mVKRAlCV#DI;uHB-p-!szAXAx7Vv5H%Wv>0*9y9t^5|S_oC6D zaUf8`;$~U;)cliengBmYP81wL5W;qbVm2wj_5qTNlJ-wP6GBq+!P4+}Q;+s+eSrBM z2L_qApKr@n04v5qjwi@Mx|JXJPy2owLUNB*Clr5_%f%-cq09Q#mcDXxNlezRpuuSu z{d!Qa4!C!a)RNL9pIu2q^7VNR2YR4lhpbgWH4P6`@x=7zJphN&iOUgoKwrf8!PU1b zjQ4-?lL?$nON}yJhghrSL&pCZ#^nl!dc9)k_--YoCc4ZT`-KW*7WLn7D%ae#z@JZP z0hCD#KBuxDLC5mnwJ{tPnjSjK+k=DJug>s9WBUK5o8)dh;wSh5Nt_=tu>7mW-t;_9 zRT19(bv;_7w!R+g0e<<c3A6y|r)V$5{qHo-44Oil&+MIgJ)gZd|8zW=@5}M&Gq`zR z%5lIWvs=%WOd0eP!7DWZ!+l5b=0z|(X+uFFea~v=g@3lQN)PJJ#zFGyL8D3E-bs`c z@ScrF=>gsWZPeU(OogSEArK@0cjB!e?G-S<sr9R^pg7U)Ful=wwy#M@gcX1hBw4FV zMvr6zM}+;TCv<P3(a+l2dfZV2&!|~Wz=H)1Qk0UFElJ>AnObm9R+A<K9$#L9QxHPh z4M!g8(seYN@I`|NGpovgy9M4;@!yK{_U_DfW{v&tVL#u*umc<1dj%yr5H*4a!9%J~ znfFe}m$Y1Eq$&JSeLMmW+p2*PfRS*rB__1TN*gxrKM8Jl=pF-a_@=bu>-)3Oz9D~= zwvZ!M?3huulBs6h@ANNJ!YQ_k;o_NUfxMiCUy|H?eG!3rcf8tNG7{L&c6Y&vLn>WA ztuoX9&J}>>*n{0?={jstkl{WaEb#mrmP{^N0k}1!)JmH1M(A*#Hj8=kxV=DlKCGRr z)l3ESYTk_IKuST?8Wm7(*ScnGY<%<I)VGu-)2*lv3byj&iUL3+8Y#e)C-&PUW}I@0 zRZaEws=1DXM%*YcYSnsbN%LcR+T1h!(es-b85scET{%2G4Fm~fjr+R?3cVvJ8|35R z%#y&338_KB1i!xl#MRXjlk$YKIc$MGbNzQI#hZ4beOAi9X?1@0e;a`_DZ=55d`&_! z+Ov0(nVC69bAcmz?*AOaNF2%kXRV&iW3ABtt<(OEJr5}Rzq-XS*%GjB1_iIlUX!4v zrhfUau6uJpw4WH(l)fMGIK3zA8hEjDYf%2bA)UPw7?{I1@a2@fJWh=qq>y&o-Hfbc z&l(h5D>m6XKdUQoRKA#c!8l1hV7iq!P0H9r{L3z=>bI0jY*%IoT(#u$moh&`GA0LZ zhVk+uxX`@-ANq!eDrDMQ5;%Q@nBR=D)U3N?)}zJ{bl5R*(tU-YIZQg?6yMlU#pu2G zZt}1?H2n5g-o5O@Urwj}(no9C6i;`?p1@55mSp&R=TjOMqWB~x0`~D2uah`O>TpuF z^0Ose`*JpJs_g*a46nj=2a62*fn<1akz|PVw7XrxNWJMGJ?k`OBRyewKyYpK+L{FL zyRCHEKi&^ezr$<h(Yz_Yak@lYT$^Uhws>NcRuHhe)E6Xp+LL|SSVR25*qm<gZ|X?Q z?7;H>%FutOuW@>%>CbEF&9QvtEU7PzrJH|=$7Sr-Xf;Ntvm2!&nzk<l?LToFgea;@ z9d%zhow5$xp^$ugnWUCS1vJbO&ddkR>X*=thG3b#&$$@FdhzS!v$q8UmGpa6hYFkv zHTg5yJbNqWN+N$mliTj_Bovl(0z7#~^u6DoeY+yO(X$V(&WH+=0p}j^ogoI@kec%( z_EKO~3{6Rl@ii<qT7p`b_#yPh<o?Zb7n8TXsW+}n1|FIw8Eo=3&Xg1D_=2vS0|!}) zYWQsJxZ_b1V@HMm4BD}9kV$~Ri03D>JSgpc|KC*wwq8tr;<Qn;Q6Xm=wGdLui+BCb zZ|szq4AWm0pmciw{H9Hs6-|gaP0jqSv0v=Dt7ngd$Udwq?6RyDR6UyZl9C|_&L0a} zemyS$ruZaX{bUTxl{y)iT4(LF9xFfw5u;HZyg@;64IA|mHEcgpJ#HgdEx-ow{%Qvh zM$CuxfZuEMEJs9I&w>_Lh&hL5{3;fLg9!~U*a&jE&LN$@MagTjU@0!Y8+G*nY3sOc zM2pZvu-tSvd^Rfc2`8uyKCtl8XwJ7wox)9)JhFXvQz!RlvspsGQKS#oiP&>HS1bGW z!Jh!rvl9TgLIPJcwL)dev=Zgq{;v*x!zv`2;~eq)HZ`MTh6n}@BO=SA?)$dOQF1Z} zB+1;LfFNsYd8OMGC0DMUd!zeOU2@MZrd$xc6w`QgPmb+k#8F#zpnDQtdW)Udk#2I> z`2OsCC2W8@4g!I#wTQ%BlbaWf2G`1RfCb+MPR9u5Exuj?<YKe{bnpaaaxp)d6Jik1 z1>|x;K&pL4Qg$`Th(vG8j<|@Jy&gE_Vs&~U;_%nv)BWK>>E<q~r$>JmH>6J<GhZWV z5el2F1=J=^%Il<hWK-2Q(mh4W;m2laZ2bm)I*Qw4hUhXdyXf@-)up=t-y+POKUd4F z<9N;h6B@Wur%Sv^{OGy6xwzgKVJzWX*C)BtwtWBK?_!Xh*B<`j9g?pVp!e^Yp+MIm zYejeN+{HIMxhv?h70tWAY0lmK<pQNa8?q#@xlc;#28h?iC-1gTX_w1pKFv{Sbe<GS zR~*VU?T=Z9v5=T0jDLW?dLzL~CIg=U70Odv;dHH6IV--L+4LPQH!p(VN!BVFP2T#` zqY+XHoO6#GFFps_0URX69JjKA;}6K~2NxXJ_32A|FfUp@4{j5xc6uB=3-WHTK0^i! zH$;1<r>DnEIujD^q#hv#mwuxs9Bc4Xp_RiW-nDao-#rMMH6VDj@-JkaYH9MY`>;g5 zNE4^I_8KNTC<yYP?RrP;6!p>^-qe@6gMyQL>S}87TnnacbGvwj5ZN1#SUp(a<O_ly z>++zsX+s-(mCe8$Y`_&~$PT|+w5{4MJxpsUfhlPE&D`5KBgWJNG>^@bH4aB<SJ*$! z8k9w%t&8z6;rfCZH+T1{-{sL95qW=n*98U`kUZ6G=L#7F+$d;_R2Uw#eaRywG59tb zBTf8+CAzm0-%>o4WeH{uJV}t=%4J0A9dAsYVES$Y(ok9Oa__EWj7`byXB{qKKT4Aa z%N=OsU>I2_`d@x4t7@rf-8d_XK}q%*xw8OaA_@q~B)nopTKWAa`xrvHw7`{HxZ21| z7d!$3*U{@agV}0@^Ev3Siw^7G#h3sZI86HGhLu37d;NGFS1);5#?Dv(hEhL8WlgY< zTGZ!O0#|S2I=YM^v~{Lue}LSoTr;lJ^8Q47JCS(9rDn9}R1c!R^__J#8y0myozwZj zXqQXH(OgUtpGEEP6550EuK-qlvKO8dB!d=Q3W)NMDn1W;xE;7df{MIk84V0j4|3RL zc7GV&-OeAm?VJL_4A@&Bl>%ujVMq1^LE=ci13~Ms1N^L1Pd6$K!hQApX!X1`7O66r zIC=T`16GY{2lc{1okf&C@BAYz!~7NWN85X{aT`a^Zx)?ZyYmXbbQ@<>B(DmqT=O&< zpwLqIvi#bjGJ3Yn1$eVc?)d`_++A~nWY&k3lw^qxqqno?)bLA;baWspd9+;0g}ew< zkk17P*DxuQn!38>0=rlsD!ITmdUkbVqY!!Cg%12}Q9%9>(m}%Lv75IyhqS`HYOgF1 zqqS@|tcEMd#@VJ}lRNW-X=LM!KXXLDP$M~=+~Sc52?@BpOr3XdzBZjFkvR&yUZ*vA zFoxj*kb5-NQA_zuvoyFuCHo{KCAs(V$6zRy5dqQZB(-W08ZsSmV3;jtR%boWJ?E{X zUBRvF$onX++fgHFp0!WP$E&C#5q1Hjk7TWHEn%3@k7M+}Z#pYWX>Q&{{$vRHU`<}C zrGjqDk3{eJJe6~15lCgmE)B$0U1EPRKZHK7Fh4ZZsW+Xs9)M@cI!r<kJE@RGfr}u$ z#FgVT0*C!Hc2U1Ib2)jC^REEcmfjjs#car>T~4UHM0bD_vmyYp6Q^C$xP73G8#iK1 z>)zYd4;ppBIB4o>kBI6MkJx1#@?4*o9jaW`I)&)3K&3xlMNaTYacD$-o9hQ%fZ{6E z%_~jDgBhPXDNS*r36|EwU8Oi(6C7@vF9Us1Ip7gHR}C*OFQ{11>Bv$qXwn(ycgB`R zuKK6acqPUZs6%DdlPoN>cwRby(;{y-=XM4Bb-Ga<*a~GgMDf+SZPl|*OvXx(&M{^+ z;|N8^vxRq-Sb^I|EJ34>OJc#O7_X&kjOokgR~{7Xa6w-1KWO5U>W}<!3VF#k5@)id zGyfhS-o=1n)e9X5w!V8QzE@0PPpcL?BvndRM5FWWS@>pgHC}&yQ$ZFtMmp`2^>^y1 ziEleOpC%-Zo>onjF2CRkm+Mg0A`fkz%OkhZ5L!2_F5l!=V4H(hDS<r!Gscz{WQO$T zj7KQi^{GEkIWFx!VrA8{Ygiu`sZ4q1`g-^scdwvo>Nz!4a}UaqwQ4$ZB>J%*409p& zdzY+(89z-Y1*S_(w`XX{T`Ni((ljRuE{@2bj>n({Ro$Q>JF#(Lb!F*Y8|OQj|9A6+ zm>Z9`a1S^YhhR7xZ;e)SC3W@o_I~^J9Kghr5W0>v&(Z3&#(zKW-n}y^PpMS8d9lZ8 zdur;LCsQ$q?RcLW8%k7z!zkjlWB(SAg!S8WnzViU7VI#JDcc4?-bRpWXUZ;(?SAV0 za~=w=L$WvEdc~}{kcJqV(Qhspz}MNvmqm!*Y|CxwH{o_}R6Zcz0Bm*1J;JL8`{z|! z0eo{<iyIr?@I8ltcpEx^K=DBc9?&JEsjy}Zcbz(UtAU_&>hemqpS)4BkcbVZZi2sF zQDyR6$CyIdsmJGtDlHx*-dEupZ|evO1P?{~yfmBW+}vCpx~jG!;8P0~E~52uokqos z)!QftCcWSaG&v~U<78NMp;ML{a>=OcQ70ex)L?DN-y$o5jBs;(x4dqU=$TaiFnU(^ zJL|0#ckz#X{_;h9qKPdT|APi-IU^%1Wi=DKQoOTj>>1tZIslqbcPn-6$KFXsX})-F zE5ADwH0-S+UsScR&|xyWOu$Tyi7A6T%awYoAn4}Bti_)3_K67-HTmYU6~s1VA^4}@ z8h~aT`;==&qjES5u>yOsRf~({5O)Rmysl0i{b$I|m>k!dn<kagpx~fjO6J1i+}z8+ zPw97WMb9dr0E@AgS1%`BOT0(d>Bv|>HdgXxnd3V1Qexk56Rw_nxyckqFD>HYYI`f* z;-j8QC^wpJ+07@6a&jtXs#7kaP}34!AQ}yVsm{gtZAruAd^R2J=A^aawqNKXu@)^) zP){HMDjwwQ{9rqSG9%f3SS~%NZ+&B<7+YF4)!!fIzXKS{79WzkoY_T+5WiioU)MbX z{DB^IRru@Q;6BCmaU&;s2FDZ_O6dYe)PJ><*%AES6eXy#!!zFAk5dClAHZK5{q`)@ zy?CA&21NWdoUHfS0Nqi6vbZ$fr^6i}hk*I|92^6n6fNo|V04yqfK<U^N?=$PtZ6n< z-O1v=NB2W2>g)Z8rvYMbEKIZ93+&Q_8Dc7fueWNvNL&7{9m+DTO0QL_r+p_P%(K2U zSzR{V(@#WkX72A_L8Z<GGF<3N&)-g|vg<AjN}h>*jg!h5Tlwrcf#8x5Up$5fF4AM_ zx+9@e?Yv~@1i7_{zWs#LWAUg70Zn+HEvdu0!+ecr7~a)T)>yE(e7)d<In7yow(p`& zNZWw%fWV{sS0{{0h%mSI@B>gfN6F>Rd1WA!RDKxd|BN6T@V1=?Wyw}rq(4s=BPD>v zU|(O7$o@~|Z(j$luc=`i+HUY|H2L}vKJKXO1$rjnkcNQ|VNHF~x4odJXPJ`nH%8@I zJY9^kM-c%1?OC5!i}dUTD3a!TPNU0aMc;h`SPs~`dr@XsBri|A_QqDB_^FnBdI^(8 zs70EU>t$*W1#_AkmoFr}NT{_<J%|E6_#jQVck7lBW-q|DMwUuXWXJc4d;EPKVnr<X zVO@1*S#d~#f<iuS3*n`o*J)cXaR{u3EtGV2KD~!#H3H%h<<(w#u8nGEsFx%Iwnhuw z`?A~<|CSuy_Bk_p);o(mAHs>yN~yDnZSA+wd4;Gh0R{v0^<0J)sX=N1e(17v?GKlq z{U7X5?O+{RXVjl#0$I}HHn_k<3Gz=s2e$|+rYm(AO)DV3{BCt9@14MbVljRzz_j3+ zq8eweZ}Q^qqLp-_TE!|8NDkLLJ7!(zI)K*vud~Q<>8{1n8|Fs8ttOw{SYY9lBLiON zF$!Of1t8e698it1)T}4Y{Z16P5<VvJ4_mP;G#_nJR1#*7drk8rf4UL9TyX|pV`;p! z=!=!!=9wuZd0u+10dODDiW;wrPlQCjG>{^9jYN+e(mg<KFh^uyWW<tE0ge`|eBMvV zNOJdeT)hmX7icW2{}2VF7jv4fz*hdm#L+QL|B3)`KjaD;ur5Pk*Zv1XyFbTyITs73 z;1HFLNDF?w&b1A@oi_JdOU&Z95fDpT%i+N>Ai?^m2_&`d;s=com`+RGog0dd56D_6 zv3z-vs#q080WX#BbD?e&isG<@Ss^&nlxqR)cj#x>DT|SDUV`{=#E`+fhQ$I4K7=s3 zlOdg$`jH+1wY-)R;R6o7Ee<2>`d0Qq%8d=r;w3b1ZnKg{2^l=aKTydQxRwJ8>SgmL zo4}GDb}z3|y#!3nkWf2NV}S0;u#tezHI_P|y~e0Hle`7LsK+i)eb<bwlTXrL`K$Xt zN;&q$OSA35Kd_4A>$s~)2+yxu1qFW%{&Lp*eXID$AdVbCK;x5z0R0aEkkzv$3#$<# zKJU=q(ldj5XH@q^A^MM2E<fPpd2}zgR#sLX%f!D62i%;bj7v^jGUHbPN;OsTB*eV) z)k;s0g5Ag2+RkzPuTN*S;#qn(CUrX8Jlg5v?-I(m^7DGdt-VGw$F!cuBIxVdFpFRO zXFKg~#vjH<{229X0vABCy5jaaBXAA|WvEU<O#>-2AKYf!N=SQe9FJ1OB8>^4c6SZO z8wrt6efbZ4`ue;q$9U@ZZw22$t%yG39N!GSH8;LbDq2UXyyYv076PF7idWnUnA{8S z2{N8<yqlyLpH)kTfxk?)qzP%jXVIo=czkE8H`Gm;%-4?ZEZ7P)Z_8wkVzO>1ucPn6 z!|;12IDaw4rVVAK&-)66h}`43IhN6HJh=C6kyHlBT}AmyED1W#F-Z%;05b$T{2JUi z5=GYv&hx~xOHt~mJNHgXH^zW{Qr4oPm2y@wMxSFYIxOe++?++y<7*-ajyyaJ0QafR zE<C9aAu&U+t?x4KyvNjCWM^kbG+H{z=*}k5s*6%Cibh8gp>f>fYU@Ri(ylvBN##ys zRLPcrp33Vk1}fdNq)k`>1%V#DXBObn-EH0JU1il<l}Q{%0Ma{XP<NLBLyFPrWp(w( zs)JpqSIY{}8y+Nx4l(>mp=Z@9Q{32F*qYnM1Kr(}98OZO8g^?m@1jN>FoBfKM)~2e zv<07uv5#pZpV5l?`sD;m?QushNAbdON9>}2>j+*mHK~pXFIY-gp{JwM;Y>Qd=y{71 z*R)amN}H9ebv4wje5=&)fb$ENHks08)}xENlIp$gnmd1ZBI3b$J<m0qiL%$weaCZ{ zQ{_;v1*fB<>xM$#a^I1GrS1u9QP4e7&8klU401Sl@2>m{=K-qR*eG@H%i~4g&3SKX znviMH8<<kpvpx^_sm@+O14fh*$|52%zW}CBuYxe)l-*IM<hIcmT<l&$#}m8(_iz4_ zkZNR3r%Q%K%{U00bD?_0LKxJmjZe#-`T@hPG{cV+aXQa#PbgE^Iv&ujv*9hl)e8yq zn*h{KS54h_SO=?x|LR+iDjPqJVBS#_X3D!fnGoEs#iPMvT8u|ta8<hfo&TfILaIA7 z1*q37Mt5_rDlq>3f7p7<fF|4be;7pt1=JfA6fuyLmTp9(yJLjl=mrOZ5*7$Z%aE34 zw4~%vL>i=Hgi1-vDCz$(yzlStdGYL(vhAu}*Lfbtr_N{-(kqu89Ob9ibuKL$233e8 zhKf;$b(J_&4=yNJD3=Vs6}#WL-72HK{RX#tIYJ;Iv60);u6bDD_LAmRJ-23#*7Fmj z{p|z5N9of<P6}bV&rS;Ia(HT_=@_t8PEhNpiG<Rxg1*)*Svx0ur>BkKLKa}qRTI;k zQA?wnbS%&8{zY($q@cLUC=`nUG8xGe+hGYfLhl^=M&x3KmRIwvE#>=LS+OIoop*AJ zOOQ_Q=hq@N1`Aa28IP~zDRZiq6Dgr)FZo_EMX5ziR84|Bf%T`bkWi6Bb+>gNlqwlL zOYUwvU^=cFCF9XbbikDvswf7hUjyL7{`vjs45PuZ;o)H%PD#QYs@Ql{9;9svEJj?g z(E6T`3|z}KQeBXXM`)UB=JuzL-v7GHOAfT5QA3$EIDx}kBeHvAEYRLBIiV8v>bgmK zd=NF=({_7yMaJSLZBs<>W2M}Wk&XJppV_atac#WeP?c#}gJs5Q0g{#}QyEm|FF|>$ zVFsDtFmO1nu;*|?@u$zGzI-&lz|P!h`{$!$MZDK{`Lt;I+jx>dvOam&|Fi&cjw<E! zYGSCEOQUDY?_QLtTUpan7yfTnNr#o(t_01;VUl1RIdA)}?zwmvtEBdC{-=W|iw{!; zI$^W?%1UXce6PYPRrBrn{t=z6KLZ#HNMhMuef4dEXS)A-`co@D8d#~pKv$Ky@X@i{ zb)EAk01fCd)I7`Yp()Z<tV!_)DlSwnW<7ee$d$Wz8H@)(L1((wv_+W7rCZjSJ-oyx zb?2&J%`Tm%Q}TUtN}&!RFPap6a(m~p!&|>S?VaQE<zl^-ZG_4^%)y$@_yqo{D!8R( zu6(bTPb+Y!?9GXfXv^~Ph-WIj5I;6w&9lw&x6BqvoEep3b%~d}l2v-RNGvy&wf#ha z=4?RAQa1BA|HR5i7pe1nG$9m~cbpRcGMMj6SDdlGleP+WE_)twi9P!BHGq`K8FS8O z-uQ5{_t3U6WG`B8KYws#pa1!3q9J`?T1U=YME5vWSRlFEiv1f>=7Y%_)G0gxG0}E< z`_|MNMkAAA5?x!Vbn3KzTgD~?+4H*{*hpCC0s6CxGF1f-kCoeHfn-?PUM_!r%}i8O zbY#ujs!?sCeCa0XRNf>&l2(q`^p#y69kox=L6aXfG4dWJ6*!f%r!tOzv^M>S0zQwA zg7(BGVAz3o5fTy6+@R3pDPzgKC<Dcxrf|;w{=U-em&m++xQCaQ3oH!WDqwL$P?e<z z&Q}$LW{nPCqI;N@2x<XU0>@5St&=LE7ZH9+>V4=J?l%n$4V6zcqL>-m-VC^g&wq-4 zT18d)I%e5rSi0)bHq!s$aBz|@2j|#E#%c^QZ1h@7u>Rg>4#Nk_9C`aq!j(#DHIKjb z9qZaJ&?a$qbxk!XV!^Y>C)sqOOU@<0RIRkqyVs`oRCGueAJB{B`B4Q;GFiPMjE{qX z&^~<UEOJ2j?~T+y6crLuAG3uCco2oL37^R1(P0%#qZ&wB`5Gy~?%5$$=^guBPNbZ? zA^dh8l;3h`5syMFdc|4-Kc+v_TarJ^Uq+U1@5)BNqxugy=dC$i<XL5y+j6&-q4HAI zj6~EY_S*|slnRP)AG6PZalu}opEg(j3;?NYP>&ih6S$yJ_ShJo<+l$o?P#>TV@EXW zAte+eR(#mXT*6O1p8AYdE867HRPHWj3cdMsRW||^Bc2XIF~<sO6(K2*5c~|VEgTdT z5i!)BrdN|`;I=He^5ZxWgSh+=ZArv(&vI+k8!Rcg!~Z~+G#b%L-f;PB0L-$Wfz#4O zTfpLMl`FFl!oklJ-@4xXT{T#@o3CR{tjPs>1y?y9n~u@bhRDb;$|%oABG+QCun9l@ zZg{o@HD+5jzw6`5ep?t8<=f})Tqb_tmVHdmdm>ph$fji2s7*c$IK%tXRDRLwgtKKm zVsGt3kkMI_LeQW1o54aM?%=bjizCMTNq7#CMK4<aQq!kw_-LvMJVdSm5rtEcb&K9Z zuk6;XWKnUefW0Yl_;^u1+-zc9&}9e9TFHcxkP_y;<+nnivz;--52Q~>y_el1CbAdo zS_75%LKWHqBYcvg!I{>ICssL2iw8keK4dRI_WVJ2X<D<~G=ZO64u>!trg*d01d{Qp z)#X$DRYZOkM=WvuwOeTu)^Sua5e~Si3O|aes^#TnP8ax6y*sZ;@0!?w3S1Auk&j+- zp>1oQdcH?h6dM{AcA7x$mM7W7wU_JAwB(1ZDbxDySo!R<ob&yKnb=>Nl{geu>An7R zF4=RPzi?uI$7yK4E#M1?2;TD<4|23As(zI5S%?#tc(SJ_u~R4cT2f+nYQ;8gGgZ!X zk7xgq@4>j~<2Ad9ouToEgl`cMsh?=cr@+H*ey;3uCGQSS&L;V=Zf}<)2V?*2-l#kb zKKyx-Yv3?yWv4FYz+*x&55Ch1i&2+51ej$@&|y*H*OSE$CZ%S#QqM6;))ejsNbXIs z@BJc~*l%0eQJO(un|35LjlS`zEUTLk*!OyO5F?OF6QYioY1+K}&fCA<#k%7~5Bsl5 ztOXz5iXK+<+H_mZIygAje$Y9w_u1h77ucubH<UL|Y`RDXBQ~1$!})~`Y3%xoi3LH; z*!;KWUSikvs!g||!VXXTxR<YMClLRaQUya>hDqIh7lsOf&sm2RTTd6O7ldu{)4TR} zcC8K_qW9o;yem9aQ7}C5>#*N+4?B)<{^4<muNH~AL1#_4JboY{a2%hNc(jAT9uh0% z;}>UtmQ);4+=Gj!es(+GZjEpp+b0nQD-`1@`$xR(O_G-U_^l6E9j*@py~ziEnhuxG zT@Z(Dxi&i)zetWhuwRYKPfr%vUS;k~zUk*N0OXT0)4wFJO9JRv+KNeKpzm@VB79YH zxANm?)t2u;R#M<Yb9h7v`_0{q`V<z8SPfS5`Qb}3E<<!w%DF|1R+&MABx&k32EXHc z)Zx^Pf23j0J+S9Q*z@B4pD@M3gJ-{EmL<2Zd_H>K0mMT^Egz03%oTI2Zu_oD5&m3s z29MC2(v#ddxK#0b4ZB_Pa@7&mO6;)wm(xs4NEj-==1-Tx@@#5AWvBPtVKscPapI7a z{sP>?d9U{dbl!R6J#<Gu%SmkG;M-@O`}zI=UjM=FVK-m=NvZR_4W?({LajHR^R?ax z5pzq<goU_@Sn&wEiq>~8e@1OdN?krWD+9O^wOmPw{{V^U*B>!qm`1p6q%0k%F|S%x z!eUfKW6HhjVqM(i_cT7gc=>(6^qw&@oW%PCq0rbNCeQ-)J+_XV0KLFoRNBX;&Fl1} zEP#Q%y9r-?un?OW>s3qH2$xPFtmYtBVrwC{vt++HcjX@-RMb^zo9b3B1Sl!^*oh?0 z*XU{Qb@j=a{%K^!7PV5D>@~+w%EDHN?ou^Sj;B-C9$<}J$j;TS>m0QiMlV=`zE0k8 zVOcKws#o~&RiXD%{OQTW+Oc;!FlZ1VxI!O(t>+4XE9kYVw<P)dTSKn>4;kNOF2Y9r z764mPIv=#Vwsr712=OQQYdq=X*pcM#uU^^lNau1w>MjfDNl;PO`!>d$4$5`PfA?Vb zjR}n7cN*ZyptPh7+MOvpK+9eCHk<3mS{~*rj~;O*?}CNl9&0<?^q4r@PWD=!%Tmm% z-rbnk`z8M3GNgs|9H}S2(A_3)fI)Gm$fS^0c(}(mPhuz1NqkTAa6M*1$8qD~?(S<k z!Zmon8Ho_|KK5vTdOU3J)0S^nF<agD#vQhUfWtr{*c?b^9Sx{l)IW3ESKu(`TI{1Y zuQ~0-f08@gSV{D@9=<eTS^J_mzxqwOC~R%{IV6Ank2kOJG(9l&ogA;I@y6(w{5Icb zJJ|juMFUS&{mx4>Yn(gCq+pBrziu3EB=7NZ6|$GL;_^J67VfLw^N6_ugy5;a!*4HY zjGH_XF&#cSLRz?U!}og|cJ0ZH$G?8uB9mP2(erMUym99yoK8JW#BkNE>ae;mPJ5$+ zi`cU7cw{-@EQw-nRhHd0Sv8(mOWt2u@m(FP5nizsU;hoqw$G2nFC2Yr;C1@|tSPe2 zD2bnVgsm7E??3EKXgpYZV!Dy0n%quB0MaW47ycoP!S)qE>wut4<{x;PgewG^)3)BU zh8=#1vg*z{Y?1T<j?ZT&(D!h(R}EGSeNMd)7U%S*N6@EZT<|Hq^go2hjXE}eVYU2) zlPB&|Ma>HR%5HR6&B}x?KF#+JCN|S1Ol&8W3+EkcQ}d&+W|PAyuO@;;3X;G<=LvHT z;a(ujZlJ4xUzo)MMjnAqI0n7LT3>m><5h4hHs5X&Jy<u*QRq!ZxQ?*P=SRTmG7Ar) z-cscMqoQ~8Hxmj?<26LwJ}U8@sUwDu3jn0YNEmE}ynrY{%h0xZ`Vo|1-wWtUj_$<z zC7C0f!h<Z9<3R%;-c{1@==&{QXacq1mlWC~4a`UBX8U9DUFA!L`LchXdf7CM49(Ne z{nX$7ZP7gB^L6R}-D1A%-z~~ql)>D&0pBDXwey15QIX>;2O0)~kO7ZHmiu+^#VV5? z&AcXMhGpzZgQ$BN$Nx^xbP&h|K1B^!$xOV9{$~9t9`YAAVM7lV=BiXJbNT)~F!F}? zEHFC3AtOSB|7y;zenE|y6w3WQP=b8|`Okk_C8|Gl1=gPGe*IS;(qYLvv8CV`E&Ug& zA#8@I;*{IJIK^w5IFQpE-6w*CDIO$d|D7%f>HHh$q~PQA1K`r=w~ZEh+wfO$b+nO> z{QtK2g#UdX_@C$u<dNkLKjCkFOEbF%pmjQb0>mjtr*ibY|I-=4f8X20EmUo-OsM&o z8|c=WU?Tw4`Z#Q#DFO0Kw1rCs(I`x>M_;>|KC!Y;MHr9;&Bd<6I$S%yPemc12$Nta zIcX0mDgdt7#QkqiejPh<5>iCYawnvqD0c<jSz00K#8y{l=ln*M*=4<2u3kO>cK2#P z3tEzn{QSrm8zjj4$aqUk%?EN6a7!j`?URJ`TB-8{Nmi9j9QfcxIqwvmhF_yB=I_rB z2lf#bI~*?OkW^#3fJbaUod7vpTqdd-<X!Il{B@Z&<%anQCm9xc2`kT{)j+UnyaoKo zuy>;itTqZIe5MUZ6rm3^ovSK1OCOYKr|PJ1L1o}WKHFlkL0fSoh6fc0C2WUf={86< z86-j{rvz0sFxtf&XNH*mcWDL}mX?=ALZw-t#*DtGuwRq^nfq-od(2npwzNJ3P5!YY z0RefDZf=Gw*PGO46r{Ytn#Vyg=JH?fIoPHdJ{N)J4mnf`tx%c0u$KwG0aOhexU1MN z@d~$~>OVNB%VBRf{lh7SaNK<XadTSv`kns)n!;6M%MZA3o;@lp3l?nSEDZ)S_WKa; z3F;e;3}!~)HhTFoBIhihJWFHIjOD-(7Ssh$C{!^gF$j6_rx5xbGV9L>C>cqlZc?Ct z{s-M&mRkmePC*_)DHPDITPK?)HDD|H((IS#xTVAifq&JSMCB@k85Bq=(SBn}|CE(o zs3A^)D6e-8w0OXm-*W-Ed1mh%ENmM0Vm>ol&Mvnfb`G{i^t4bXKuGow4F*a@UwWzY zfOIu;zW}ar?!ImYR&FZ5uC0cKAaLvK>SBD;j1?&}3=#SsN{|8(AKdfaAR|4l{$*LZ z(8!xUOzbx$sAGnZnKrFMa{935yd57+@c2pe=jQ;uI9625<$*93j4m_6HbkShBkvFL zb%28=F28rKSx?`Uv!GJje-Z8rkwi!Q8@o5?^4RA$Kk@|xfvTg*!r(0{x&cT#-QvS4 zlJPY0>K^Gm#*tdR2y2ovo-GYw4$Uy1e-r^c{iacwdo{-kl}iZ~#QKTtQR$2#tuJnd zcX~H)Q*A6-4JvBP;iirY@3v-kO$uEPk&Vj)^;n!`gwGx~c>{ONqv$@fFD|tHKj!8z znKnQ=R`JYU_w8IJZ_pRm$7)YmFo?u1XUp?VrS}bdNhNMJ;Wr-CiE$J#0#JT&v7P%3 zXwHpXmJM6Z%`SU)A8*r~dkF-6B^k77SJ2Mfm)Q+vjn`ofxtQ->{_m8GxvtE(bUQ>R zoXV4z<|uAba(g_il(5fzn4~h!#prrUc$d{>%b2FZ^%=_8gO5f=1M*I~ySp!0+#r+q zRV)?7+z#D-clSSqY{of;`FJaRPimr~=f4}e#P;x##?x3QR|&Pt&z>heWjc_<ZlEN0 z=yKP6R}Pv7FTmwmKCYw29jjc7RdG*yoaAwdgga5yc(9BeihE<6L9U|~P5miXS~(GC zE>2GW2D6-|VeU7TDjH=|G877&Rrb0X^fiFcfstcA3J9y>vdXk%;ZQdppSoY{@bU6O zT}y^m@C$$yE>g?_CE^x_9bKks3G*jEk(HWQ#mUn*KH33J!^*&*MdM`NU*`B!C9;TJ zxQ}z_6;8=r({Z0fTXcxa3r|ehm?-KhEHmyRdikjHVc`T$63MObNNU019>8EX7mKs# z+y2yH7b#d8ge`f+-MCy-X&z$AW5&|VE<=Rb(ydFkP|q=0&r10WgoXt)$*gcwsU%9F zQvxMjBtXCJXlga_3Odej57?Fk*{hVfcdg+O2dTRm@9o3alEvn;HDkU99n=$D%wZ(T zLmr~I{<Jxb8d#7`{L%f&a3aWS>ST4{n0GHL;xVWy{lS^#*U%unQNkLiU%2{sbjdwK zf?OW__JB5C>TM3td|tHa!eeowv+sKXKxk*|>~Ux*_NcLsbt|}XaSyvCcaUBy+?I9) z&xr%BQ~0s)8w({j#gzhizQ~A}hf&XzGc4IIg)Byd$)>${L5zB#q!GNwuBCu!;QX}Z zJ%}RnZ!2bRxqCt?-1|ENbCaPGDmcfi6>~R|hq_WzI35fMg97IbF3OOQkhpr!3jE1^ zm)KqIP$pN_OAx^r&K3H@<KsZ@c_wKVs1{+lVb}RjnvRPS{LK!ore>F~)6!l%rw2~? z{t`j^i!!!f%P4!B84(9wB+LZs{8eWRbC?6}>B^)C^yx)8D?W1D$5dCuGre)R>rDQ4 zT?%)LolGn?1nc(RK>;jal%Z+RSMmnsV(cMA>y;G4F-cSa>wb4w_HvVBh=e31Vbssp zRr0B}Mt|&y>&OI#edI3x<Cb49%^rhL2yTUwd@zdoT=h>OzmM*7tjHiXJbSR`Y0J^I zMzt|NU<;2YBA?LzAta*9T9<?FPwwqK6-iV)E1UXb_h5B(6~;e;)b&y>IQ&LcFUo)b zai>m)$U>&_t!s2YJ_qs~Gc!58=SL|TfOVrRNchavZCjRpSPLn*p;{`T8Lgjn)A2gn zdg{mHX**UdokXJR6it4DwArkM$uvXA#xX_zQ{^W(_EI&S<usZiIE8(786KT}TgVp5 zd7{f%yHJIXk;YEwQSfX(FmU&P`Fxoag;Qku_}v|bowv_{%yHF@DgT{-<_Z9ygLQ7I z?^RSa(~}>4QOvuKwWWYGD9(czOu>~)wc;ffPbVgZhv{zXAE^UHyxOEQf-4SwoD;tf zveBBwSA`RY#SN|tbw4a7igv$2?lG*8%$IMNUv9|3)KJeK)j0~$6jjX`wo(*_(XDBD zb=465w#hhF)^qi`e+nqux+`d#H0b|8I?a@Ox}I_QO%g7<xvFKM&=E_Xv?y`%t~I1$ zj?#)~nP`iBhI<whlTrC&Hu46gv=mo?9%&BAEHC5!L#PRmL?}Y3foGIDpVxzvh*dTk zH0BG`3nEN+azD{;0+B#$zKg%dHVdi&d;<u=JI&|c6<RUnKW)L6O);XcU>SroE^^q` zGBVgjMyjZeacNwPR&vzI?Vr*I9=bUrX2M+rl#c0AtuJJj>0=cfhrKgCGj7%G6}Bts zIcv@53dBCtiXGIkJ%cjV3uD(g`y_)})goecS?M_w8&!?xvrB}V_|(d>gzL(OjKQ4) z4+-aWDgY6C#@F#PF_JqLZ3Q~pJ(9bbF+MlCiHRP_^w0rx2eyLhG4z!&E@QUPNpA8e zww*dI-U2Q|nQ=UTgtaCsJtmbxq2925b9ky}kVk;R;d_yj!bN@a!yQx;4?mU28<UlZ zg{+mUqXaPLrUOv@H%LL~q@|1zo(&D_O{0-dw%_E|Ze<WfyhF2ek~TOmkIf5pY6c?9 zM0GeKka3xhUUS9MHOop5qArElhrDWS$VjE@jojGXuY8EXP$VTKp&jYkHUmXN>94KA zs@~?ey7V2qi7A7?0(vVLs;Hc%k&bakyiq?@nD?w_(o07^n*0T89b#1#{wx~jK#e~y z6)h}q@n)BYcWTno=(m3?F4|+Rbx_0<EU;2wwab;yiiSSCbOpn!6&Bh9T*{=fg?r)8 z&zQ83$;ZnG(&?{yCv;b#2P_hq{6I2(NESxWm5V(|{lz+Q*h%8^d-poyqr0cWY94GA zLc90`dcfTqlH1HD^*kG6YVi4m2PB}}c4o`g@E_>h+W++6{E27lvz)j|;mhPyD#Sn@ zrRJC6mMVEay+%5`>;Vt(LzWi1J~xjK*4vAbz;Hfcs9P}=aaJc7G}Ja|vKS^-I+t&W zHY1BAn0$Bfx<2LN&F`&x9>$AJtFT1YXJ}hJic}uaE=Z!6bM>mst2~`m89+Cj2uA2% zDWF$mudT|;${KdTp|8a&h0K;)lM3lX7GLb|mfSM9v(@LC^fma@j+STQMtS(HiEA+M z;+=nO?e?fR84OgNVyMX=fb@4}-xgLDusN~93NB@Dz*YDgCLmC8S=YBf^t=im_88ah z1B16Ax|g2h_!pAK-pGZpzG{GH;F+wcy|YE;2=X{6xL%nJExCt<hMoz%DjZY8l<K<Y zcH<FV1w7)#TqNpbCZT(`<=0yQ>V6>3fZb?V+4=co{E16TGCfy!-C-ivpCk{ld*A$e zu6ur3uJE4b+M8#t>6H|&eDp)GX8r2;I*%>1=oND}w)H+3@IMRL+>2iX;TYyhmgfFg zq@n_vv|U9aV`l{j#w7N+q1GgOypV-_GAQ7GqCu6mc#JtFCZr8LDAcWV*`J`alH$J* zwzm=_&cidNWNR0x1bXpPno916EqixbVx6Y$*?%4tU@_OQDnLdH{PxShYtqbS$bIyh zj<s#K^Sx}+{e;-HoiYO$2JM0Ekx(uF8*fW1zHkd$;}SDykex3Kr~MkZjl?T;(vQIm zv9MEoIxE+%ez1s3L5guU*NryI=(?bt5UrwR1^6AZdR<AleC?-9g^UMPdX4yg{X&_% zcli7#B$r@1TX;e%xmho{RQG$VmV{d6QwH|_E{q4iMGv>gTI`S6t(Fz8*X{Ga82n`_ zzEyEf^7nVU^}>VAL2^Q4{GO>u!O&LqPx=q%$1W~(QSEA3T5}1J<Q=I6BC$Bx4p#`9 zL%Q$u1$VJ(F}8JC@){_Of9$$mw6fb0=!u+c37jaAlXznCNn(N}NZV4$gi|%~9_w1V zdy2oI=7lykj7tX0FZPE(z}|zl!Kx(o1h^Uc)RjP6Ye9ol5Kb30Nq)xW?bm!RX;z0y z>l_-j<k(Oxn(@7upkt2)7E&<-SWOs!SL!2`K)Ml9#2ofEH`wgpl}S$Cte2sas;g2h z>I1q;p)6wW`$XCO*NSC|!vkJf&$SQ;i{;Yi)T%y8iF^VlKtw;Vh#TMhT~VtNob-5k zy&`G<o0IsH%~i5}M$tl_{?3Di>q1ZIdEt_k!}J3{d3D`p^p@o2aFgWzl#?XX==>_X zJprbxJ#wa7#Q^w4khq`z2fAyBx2!5CtU#JAPo0m;Y{}DW+w@_oo>K0GwaLpJ+A>eQ zkaB;2-GK3lET%w~zqSUM6ZvFSOl_VH`BJ(JV^YX0Ngg0w#vl7<ABL;tr*zx&LKYw? zi1v*1oN-g7s|=nBb=_oSd=!@n7UWTgO@P8e<e2w_UI`y#RyCNPTK?(Z+4s39qtx-@ z_{9&}g3I&Xb+%;jqr1Svmz+C+ng52`KQ&t)RZO|P%}O~wpAsV(&|7r|Q5e#vk)QFX z_yO_wkYWd$Lqd)Tht)5vtI;5+f~3y>sq+PSx}u8yL33)~<^06GIS{nn_GWuFMdin4 zCbsFfIz%kzL>GV3C_wJ|(0=P!h0pXd0zjn)WPI@|4cr6qqpfpNY!RcaAkfjQxZ^<; z^qfIF(<V-FilKi2rQnUZ;+@J8dFFImaY6PWU#wG`JF>75h{PMkD@96Z)yxZI4eFhS zC<K?N1&#Vaaz$4KUu@6Jh<n9{vVY(Xx=!jE1>(TAMMeDgGta+gMj}|ZIiR3**K9x; zC+;2R-*&)zKfg-d<kI;`{vo2RJ11mnwqW`D-D3FiSRW5=+Uu}SqqmcvCM4l>7iYAo zM>A=?9rX!NuIle5*Lj@y*Kt=qs^h^$<4-r2ag$VWOD>nMYm~Y`CnjKCLmae+20k1G zX4H{+hU;gm6-O}i8XD&FD5FE-`>sD?3<sATou6W#vd)Sps*1wYFQggiI!Tgw3K<nc z{`{u;biSQExohBB@csU87{TioP~_WOp3Q=ThG*YR4WzC`x7&f$j47RI!f-PA77yO6 zQX__p$#MGg>ZLanK<5##mm;aYg+sg2>>=}zm6n{)q#kb_+eQL=?>SYGBdv~-90PS@ zP{FH_MsP&v*h_{!6kY&(X;QIb4(5CIjb8@U^xA7lxCxr3RetjO82?MPN%m1A;L9;w z7mF~<({0I&0WwSM6>U>&d`mTEF0=1)7Iam(rB?cTijsrX^!!~vQ_De`xCmic6MwFi zVzFG3*@DEyeSz8I(Rn=oX#rjpRvnwAZ7(l&Ks@iAd&F?^l8lC}Y)cuRstt;fr+U7y zFrihce{uVa$qH|>;(E`&_?192k_R}ag|SQ^j-&MTy4Un~9M*f-Or$@3xwQ2B%ACvX z{%OGq!KX_oVd1XxD!mNJd>+qklG@_{9<sy8MT|9qoxZMKHq8tN9;2&5a%~w8DNPzf z3h^iW%{~_7+8Ac_cDJI7rnH=t*jtsey9CX`@3a-CM-Xfd5n+@1uN;kKs$vAbFTM@O zaR@)jsL`RPn|H6aQ|p4&PCe~gLtwdIMrN1XkxhQ`Z(bhSx&4rm^X?_U%miGK$Q}!s z?u+G76<LL;zWI+#OY*EQe`SxUSxp@6h93FS$)y~RlyWX{QQ<yiOWRA74?SMetU&Rg z%?f^SQv~()>goqt*?ir$0Vf%aK5M?`H(9#l6R@IPYsIzga34!NqMh9-lrOHKe{!`P z=WwOexw~qlzYm9*dKOpmt(d)%7Usd)YOUD7o1~iC94#4)y_YfWKY=0L9aMLG$G}VT z)dftXo00tSDwnFtCb*JjEmmUOt7|_<JDR*=37S#>6_^(i)E1{BXq4PSO!MYfUg}(b zG%U9NfLmQU(q!yGW4-2o=EQs}Li!Ad5O(^2KlWm~t}UoO_rXHB7a)wFTgHRIv<O$P zH*7R}wjLf^iXL=UG)*6rEwq1&<VgSIpRK-2Fq&q_4WY)mtwAD6+Z1U44%)2d2ekor zXwNbVIy*am7wQDzSm`)8dUY;Dby(%qi2|j*oXP@W#;$e#W_fq6ez;h9>!=|F!Kbub z&O?yHXlxtSn#6Rj%BIm>l7Y~b+=*uBaRnBHRC#_3+Lj?-!VPjKQFZW7J0#$hJ1F&h zCbX;`9v*;6KAnhhQ_H*)X?j6Z7T`9NM4izf@WY^$XLK{x^S>b`E)LfIx(i@fd03!5 zxB+l1koWtyCT`|L03Kc!T9K4<EdjD9R(MW>F=Wa8N$plK(%mAc(1Obd@}pYnS)&y< zKezC-TsY=P1@jcVz&4B99FdFA1YPBLSD96LAN$3Oiag6A7NMTFI1;uj)zsGslq?&z z<#G2ig)Rl`Ei3UyYkb;!Npr@g+gfi(qN%BAewp<7d)$)Z3qF-KkrGj~1x0{#i}7|H zNUrR01^u?Rx><k#QP#F>S8M6m6^mld^(J)<O!V<_Kx4Z+$`S<PpktKNLwGI6oH#|> zcYq`s&X`1wP%C7oC*1>c$q2I`4?$@pzXwtF#!#ui^^WD}<ztRg*sT6eaJ=ca_30`_ zkSB;3Rw5(UBjxr-<cJ|=#rQ0^;zN+ca(y+a&AB3NeH7p$6B{P%z@JxTEPXoZ9CS&> zQMJw(bQlT%b5OddM)MZ@bA-$Th8~z|y;Li<!0*Apxh)lvq<62@M2|P^;@spnl(9*v z9&aYQ3`%tx868<2?(sX=$yCQ1b1+(|{iRReWIOUjpYhr^6MYTW%vc%f!D9C4CdKlG zxkuo~%NURYdB2^)joiHG5kNQbKSs~4@6pFrOKIfmLP{s}m@oQJ#u8nY<@j@H&C8j+ zHaTiWl*FJBx|*GERd|+rd8|5S4JQ*I7^@63GV9s-dgLX{FGUC&aJ_92J*73>LzOBn z$?$+j7^FYQ#>(1lyub?!15N7c@l7$p6rsvPiUv$75x!BeWqPeX)h<T7nqM<QkJ^yN zzum1eJ8;IRRmn3+Fuo(wDN3F$^NUsYzJy1Za7CaYqhwB_8r>mlb~UE?)TEGvu<!@= zT*LfMl^y6dh4Ih1Fc&APwSCcc8Xz5e#da`N*^^nl0RU<mi+Rzst2;PkULTEF52V3% z>&gGP_pGwYZ~94<rInTV%qu6PT&aQ<oAm&;j>}Xa`I%Au%e~rmhvLdvc~{|{1s7M> zV!-Pc$AVLTfwx2CKGL(T7~tbaF|_?i9)mu*j};Zn3+~m%SG-ur8&)6ZjgO#>n|~o% zF2pE!^l4nXe%<m(^-rO391i@mgBy+&xu<l<2yrFsu_qtV%C#0nKI>0Xa|>;n>=Qj` zVL9>~GEDZhgqucErxy|rzW;MGH8*N^2g-xRXEtnanyX?>rFnF7ydE2(%XYry0nDZ8 zQZj6hA<bn@B5%eJ5*l)8!@SZ>XFG~5LZ(88hM2nqRtXN{sl77){{jTA-uGG-a4v9- z9*tA7Olh8Zj8=5CA?T`9ub>yITH2#BZKTeFiZ6lM$E`|<DzSTe+q3v8TkJ#}%#ocO z22n|)i?f+Fs##haSRVS$QAU|foz*=S8=wdB?nWY7)F6stYnRP&^ZR!|6)Xf{TFlT? z@cw8HI;5HKlzfi1Te}FLwh5yGQIW6!cYZXmF}8Qb+KOTQQA00Au`VEhfJhRZNh(+v zzT?Z@2C1Ams%*Qsgo=D{M$$H6KJ=xZVXZ`8?Afl9yi8tvo6~r(pY-KzC?f|q7N2Dt zXG~+>uA6YID8UJm)5lk>0iXV+vaRhMYE+I$5!6CezK1^e3;a!HHsEM6#>5E~-Q*Xu z*Q|6oe$4)619$P3(3#Y~)remkMua(bPOgK(U4{#%sFuBWk5T}1ZyZjS{mHLCo(n~0 z-&MG2?8RA3X7I6D!{Qvf<V5-6rV{NrA|ote)*e9ZL*kJ&QN1DPfC#62<nGhF8UHjq zQlzTKnEmR@9x24Uk#et5Ij1d#7~#Il7)nA^FmVKnw_}KJ%J?&aj9%GZ_b-nJqXh_@ z{ms4+lM)Tlz|aq(dWOkk=_jPVS@Zqrt@ukY687Y*Q;;%FO|%^MS0Mx;N*5DupMf_% z_9K(Df2wzOHa4ErAbgVOrYVq!mkDGF)fj=ztf%<b4n=s=5Ank6^bzMs_@6VuQBL@W zeK=@H?F9ls2tibigS;IF9M5ll^ag=)CcGVblk{CTVRr~80fI=P=?-whe>F}=9~h<i zPrv+np{lyN`d`%upGw!-<<j>F{}+M+DHkO|p$*)8<PW2s2mqHreu{_jO#w))!F5(J zwggm6!9sAH0l{ahe@d2D1+q_oy!%#O#t5mk22?5aRPX=eBmXjJ*JL8hE=RQ>dDjqN zQ$&qTJ>`b07{gRyG+66<SZ5OlvklJU2f#}NO`(Sfh?Ie|fQTK!nywJo*E0=5>D5>K zDnL}iv`cW*mgIs+5uWh_y}^y#YFkj4)vegY0np8&Q+bd%<X9oz6(#z}IDe~rXgQ!K zQRYZ@MfBi>ybF=VF`{xXQA20d<yLZ5kTWP%G2@n`k|$uty>sh5#Yu`8lrj+(25`W; zoCyv%sq>c!M9~0_5=0HESuvL0kop2FJRur}dbo~V(&N&^7NZW+M2+T{j&liP_84R; z1O*geZ@1!IMGJE`#;LCEsz>e^SMFN(5M)b7KZk(aziOlU0bUiY9=IQ9uV06tPeoy% z4UgVZA><{NZ-^I3onJh9fp^Jn97S+)IV{CT*I)`3;C2k=1f&-n=nz;t&x82rl5@=d z0qcGzv@vjkxyJgSE;+h=z(fwCBA~Z!jt-Vz<wImyq+@^3yfg#$eoz#du(DgQ(CJ{= zlVOstdxhaQr?YW|VZLqL*XghB|Mz2Fc_aIZ5C#6Q$r!<#c5?h0PaTU({eK@Ql3E^N zIA5Q5sp-STQ*%{5{oV_<rfyGAYrdkek^^HXc+v^cCnf?ZjMcpZ+D;$F^YeWe7-{N@ z*-TIm5rdo8nxGRtN;0|S7Vi+ML!%RDsc!Bu9jZQ7ve0olTIikKlLrMZa*P5f@!UKQ zEj9tlCa}i*P<!r5u#^&B_5C0<f(a&CQKq*P(oJ>`EmjUDuL>_nPlQFOxD%U+OPn<M zW-j*A`SxvG*<MwJI@N1A4oS-8V{Ihz$6x8QK5K9wLr1OmkN3z@GVh~QpHHFhai#RX zo5Gh=Mn03ixqdx_c=-72s8<5hOSBQ+*FAaHo`qHSwq;Afh25r#-AC`Pgjgt)Be&NM z=nF&G#n;+BlCFd#LHKnP&6O((43vTQ&5=lkXhqDp3D*1rOYRMs1D)PdZPtt+9riZO z<TFmW`s_Os!gubZ9K^{p{ERVhHTH0H>@+pJ713;2644)=_w~z{`>fHLz56l)*x{M` zB^?g8jYgWA<&icyGZ3Z1CQY5LiRrxhVp8bYi%5BfpvIqQ2k-{p-D75A`gtzaz75-! z^I;p8e(0R--Hzn3ZF-nW<Jj&HR?S0BpWLPN;lO_+<+mrplblVZYpY8Id@xsi&|eqK zNl8f=i2nsF!}}}Q{Pl+R4G2e0qXx?klU6;c@N-O*r-iLz&}hC1FZ(;eFF+Z;(qdU+ zES%<ma~$xmVDk|dZY%!GZ25$(>s9awUK{MX`}D4Ud(_VTFZ)iqBRce>i>3z{{ayWc zuC^=}_K-LnWyZq*u7F1ZsSg&k;gTh^#Yu?_v={uy5sh(~w;Oj%^R!v<7(fdv^Ocz< zxx{Y9w8=QO73W!Y;lW{7Q$zdMSDC+W`ThX5wxcolwn}fQYbCA08h0-rVmM*ZX)3;L z+Ev+>`RV(9Gh25)eigQhXCnHo%Q=h(V`5@-;>r24IG(8YYXAV*kxD~q1D3YsiGOrL zY|Ci2eSCQxW&Vf-4fp=G8EVJlUE77Cn_v8tyzN?Pw9s)^V32*p^{!}P3XQ0vfi6$% z8}i_ns)HNwYA(v&hd{Ob<nCvJ6+`|HId86Va6Enc@YK1o@t&9L3uXDab|qiG<%xfJ z8rCfLSuSf>=C;pk@C}etXN;)5AL-z{KlGmp-W#0wd<(loMOkVNbK%ifw(;A0c*4sS zBBsPsejNNf8->WU*{RUMXlwgs+k;*O`U_CM6b|ltSN{l|f!uf2*sfU^-M?#96_Bt3 zPqtT~i8+#g<+zs^Oq7l!WZG~za{AAApN+}%LhUDx;Rn5H3(l(SKc|Ya?bwOXQL8=t z_-Bi4lr4Y6?+a^8+pDyHJe#%m=+DKPUH<W1$wDzBa{jfR=ygX_RRO@2?hH@;;us~( z&xSt=%}p>UxhJCL*pig91V-D<(nj0e3r}K|WfSF~j7hZkL7pta(o;&_z!I(aqn9^9 zL%oIk^kagjmj=>W;UhW!I0_%^1z`OFE6tgCww|Kss9?z!tBh960M-v|{B2uSh=(-3 z-_vzdLM#2gFf_-PnPIGETR094m=F;Z7zXQw7Q^V3(-$xGvi8aCEu|Rd=j@sD#M-+f zReImjvmrEM@*atdwaT<iyxEu0$<-+fbZ%{(kVlqUypOOCcEEuSWXm9Ybe`tWV5Gv+ z(3i0y9@X@{Z0McUdGqlJNscTd&RnPWmK7K8CaYI8a~zz|S#esel&_m1TFl<qKpj2| z^hR{uOL@OcVYsp)eIxR$+)pC%cYpHQ+uN0qSgU{*i2ph+?hmJXCu}O$wC>r(>UpIQ zTIug<=KQ&`63I@rz##3tJ6Imzik?@MFfx34UFKSuK*7|V{pJX>w{w3!kDWa@<_=2l z-?vf0ls>GtnEVgyO@;29=dEh=?TcG_ouj#cjX&O!-_fBG{UBJZHIcbugp+kN`!h-7 zgkQYH)A$WJjFQw%FIEnk@q{1~j5Z!yfTvyMUmyI0ou7rnEV(-Q?mhp4cyOg(cCh9j z{R-D>zZ6~Bc<87<%9g#`c%v+*FnUYN#a8Yo?gjSq{Ka4~+4*(yw^W`Qnli7rx)&A} zSPvq0S#FmOxD9p4pZGOof377TgrzH8CRk@i1;-_~1JO#|>u;8Fr*zZ0N#DGm6(#cS z^bH<uRr^+_=(rSEC`=^k)ph>edfGWQSaOZ_orwR`KoSUfF3Yp3P%c}K+mD-$nz^#! zN%D4tuWU>66chJJ#uy^``zT7VNvr+V83z#`M(oYZOPhP*7>D|<e%{(1Q*p%f{0y^v zfYrWgO(h%69XyJ;`<dRzp1CF8GQ=R~;&stti<scU^Vy@ykJFX9q7VT5AH3+me>r%o zjO4XOYrJ*}f~&3`mvgtovT*$SxxyXB>&cu!!pG>ZZlkRpO<5Ol;EH_fd)*O+57I5$ zxxBAT4emUyTcXP7%(Psh!`mC0m^co^Vt!IxMi1HxVaNsFuq$S06QhEYCEcp!9Z^4T z>0Vn1cfeh^aN*9{NiBtStZU`yrg>Ji6Nm}S&CSp0TqZS*njCDNzIx(rayUN@XHw)f z*sR$TI_cbeN|rN5>)ibD%VN)({(0H{JJZI{1N-O}7t;ZzD{h}(oucxYpi0TGq6|D) z_^`&t#^0t{nVIFgC3?0!_Lpd5McI5Xmo3c9q;8Gchuq;8PHEL&91SBIHY#a!MgB(5 zdPbPpZ}#6LX<Vymu5~O&hlwB)UmGFh`De~LsrK=Czq*$);rLZsOJj2BGl@)Su!TW~ zp)rydTJX5qS<i!{8~uIs)h|A0C|n4#+2paZDQUP^V*A!sEP_HOCk8Y|8xaGLni-$i z4a;ff<t@5i9p@=;L25SUpdvIVsHdwO_bjAN9rDvq@S`;@U3$#;^wq@Bkab2*rj0=G z@87?Ra;&D7zvS(f<*$6Xm+~z-b(1a<VZ2)=DJpukWYaiVTXV)^W{8eGzT=eRiQVvp zD)7X`v?=I7QrhH-fL~pIfTU{q-YJEw-t>=t-i^u7dW>P8-a7>0zq$Fn9rpCdJGp5O z&&4Jth8XnI)|*hrYW6%8kF3tzST>x?4%tR)M29W*jOk*FW<3#(sy)mcvi5B~rDVq2 zko95I<u@sWZ%Z{G{(X2GKdV{7-Ma8BuFa&$UMb`K2osqjYN+%lF6+@x&!m_R%iX2x z%D5aiQ^srYjHlwf)fr#x$6nts2f2&B{t%vI15I&vpPwh~g6|0(f2Q!N5(mTvC5g!d z{>@pdk@C@316;=u<$RcU2OQL!QCThhn(lBXLwOxLp%Wl9gUKA=@`yYiq8#EBgB)YP z4s8oIqbCBlpC(P|K9P4UgVoSl3PFVg+B@IvWYx|DUCs{NPVuUT!b_{@W7;@2s$g%E zmRGM%ZU-&>R3KS@IqTWm+xukN4r1X9Xm!NPPCSrgp$eX#V~O2*nC0|HybI?TFtWE- zGheiz$asv1RLl6zw5xD;!j8zofS@J7;R}-D>5BR5c#n*OJNyy87tg%1yp!<m-Rnw+ zA;-|7joM{|t+E2P<B*P7=IM>ep&|Y6<oK&MZa8g9>EvOryvw}3eQprN8~dgdU0GRa zm`^oqZIK2EEeRzK=km20+U#+gPR7&s_B`3ksL>XU+On;yE3Rw~9}#0UFyUk@!0g!E z+`PVCTGi`VogEk$c>fk1#6$Px>}~FBP1U?hN>XK@q(6FLv~xnTTp8H}1O>B#6d5QX zK*}I(=SEAuYDZ@$Cl{B;^gqzkU6ou2ZvRM`%;&gi++67Wh40)pCwokrsQ5`KY~rK4 zYMnwhwV{*izOn|rr7#s)^Ic{<y{Lw-an?%;jil4P0|L`EO@d`}#4(vB%$b6r$>p6D z<=sQ|2^F4G8L{>8q7z1XNi+5zBvwnDP~u4l{}0kR34bmx2hLjbhld(*4mz2Jt{xj* zUUdFpPMi;z7QarK`;TWnEwV4O=pzq9oa_@5@@*MnvB|owbO0^k>mDIEd}DCVU=uVk z(Vd(T?}FB>(nz&f3pXE(PSq|<T&U7~J4GL#5nNtPPk>GgDvb&tGc*k&n^NZJ<71*} z(8D#Npy6OjNo=55x=ci89slOqoGx2ozb(&OeyHCMgt(}Jc_tM(W8&lDY*!OvOn1}E z&QC<bDi^HLJ$!I~y4!u#SLvOZnVE74yq9ubn9NNeKH4(5Yc^ix`J$kJz(8Zv*<aC$ z89aCH{O~kMQGb_^u&5gaYvj7Qxy5D1W@4xUDfTGCE7`%Q!=}~O>c`PzSYOA_&##lO zTfOnZz>9WjLE@?2%~+1P!rL)z`J<O!udcc<tzMRg`xkE)tWNYR%CrL?rp&}EEL=Hl z{HQ{pBHlX@ry}{t99qOSI;}*t%PfblyU-hlMyn&Q0z%SzExo*|8ea|UJhRodCDpsS zd;KvZ^qD+YaOF@tgJLR>PrR#yy{+8A*ZD}V8nQLNBh022q%<STM5paWDS$H<NgQ|G z7WdspRGxjb%4&95Hbb*zc8$_xBRzsVdQGfwgRyRkQ`K^Fje1%rbShlLh(j6WXiYt^ zN^|1|#FkX?K}GKUC3@Z|<{?3<DWgz5GRG^E>73qBBF<1TGB|cBMm1|~yGHqtveOB} z3tc0uhqYq?kmTm!O}lmJhn>@|)2;?HF3Y--_au@l*xS}>Tr0(Rc`tkv1i1+z@S=1m zps|PS^1F$z@Iz)YLOkl&j!R_QG!-@V64`N6P1|`5<H2~2?W+goKOY?1HSkCd1^pI} zjFb>|y34+!y}dMPc-{-1{nt|~j^nSAGi=?A6>9ATJ4)OV<#U)|x<kxdttzhM2zb5} zI0f-YnKo2sP8w8J>a#=L=krHr0Y-oM;y)kca&vP#BtA}Sqj>0oM%7w*G+gswNv(dB zOoocm#6`tcB+sjRQeQ`ZRdF`NlQ)p=z14jJS9BpdLg!;#zWxk0VtFSbnmpA#Wm46P z^31ExP?VMLa2nojmGrP!xb4ti3COPKT(Vx*xB?eNWk&65o720mX~u4CbD&nwAj{=- z&6wXRGM0r~R?N7Bn(k`X_Oy~xKB!OcwV4ZFT6XoAms9=oM$2xN`^&Mrg<g028W3<i z#yiJj^XpSbiHNcbx-7+Os<}1d0ZqPCE*kY2(}A-4K{WRvgiGbX_VU-xkTaUERyuhE za)Q$gR<M&oPZ%p1v6INK)@fJCJmPdFT$n<Ed|IRw_YdsX#aANT<+LmqWBni*4t!}o zjk`RBWMnCN(Md=u^;XdsSu@(<aoe8x>}(!VXFAi{#pMG+29s9!xvaFb&s>Am29&#^ z%5s=r)2@}xsqSv_Y$MXHx!9L$2xNJYU0<Ixo=;t`ReOT(nbarS2Tn{;KUzo^)mU5l z8gDSe%v{HFT$uOpu7a#E(XVJ{pWt??ILzRJ*x`x0Z{ECVmb_IhVNo1zob698@j7-% zak;)4^|0`(2p=1Fe8*V|!mF%!7oUb(tFa`TZRX)Mte9m9TD<k!iL>1$0fHC`)(3OH zNS9=t-ugn>?32?gYk6#rgTL0Kq$y2j!9jo?6#jMtUi}|weP4k2efTz2tYlR8vpXFo zy`?Y>sCF9YKpIo|$B!)47leN#oiMAW4(I3^brg^1Todt6j<}ScewkRNCFr!-yxGZU zJ|1%EP~-MSu?Tr`3W}EYgl9LXBUUB`&X_$5dWD$dOi(HA5%xY<qO0up*xSwz3_Q() z7~(JfgC0P>*TF${=cvzi+gs5~Wik(LXi8r#BU9>rlrc0sY!J<>WsxJQM8$RN+gdz@ zuaBL%`E$5X{FlC7AwHjDT4uya!gH(KL2%*Tk_dkwTG@}vr(tx|C`Da9L({m$38s~= z_MI-Y+v9vHCDG4xb4raksX04P#+SY(9=s~RQ^3&m!}KoR3x?Hq?|$tCB4R654OMT3 zO+)U$sF89-+0NPdfIFH8h8WfOBt51?)!Z9NKalBS^%p0yr<?;!^`b)ml$hgZV8Hck z&AlIYI3qRiCGtsU^CO!N`Y>q~pU`G@33pGmtO#DL36V_Or#^qk@nJ`MiuYIO<ReWt zEc{aT0(J(vNJIJBq1F)_P3;o7$Nju#fJ-_U>x=V{$=Q5A!=po8+pLn^_Aq!>`@Q)y z@8Lu?zc&M8Y<6=SS0fSo^3xSl3qkEv?jbY*xW<;#{Yx=4il5i5{i<-fQ5W3V(iW4? zYA^3{v_&R6B)Aamq?^i4SM+h|fEm;8-7bRvDP4rT{JA^}1GpM6MGwER5?$S&PPVp! zYPPD)!Y3c_7)yqQ*6++^>IEYSPi<PeY@-FNUBYu&RW#4$@IitN3?fyRtA9U{+CL5n z;^yX`O)s2I+k<<4PEiGZUPULkrvZnt7Z4Q}Pn2h9oyLPWqvS<~NSuXG@Qy;iTL+Wc z!s(3$UVwsD-CbF9ym&G-VB{MrhfixCK%iyI1Dlk96c|xCnv(CK1;hXnYJHRU<I9&X zqoWB}|6zMwhwP~SOq~(L87FiLS#oy-b$ETMQAei{>KZqB8rzbTMK@1uI}}oi=?d@r z__*`!l};kPybN{puNg&0Z<$u~zg2~)dE~`<0Lj^?pf@|3;ey%i882n{{i(PX4=#Sp z$xq*NtE4@hCH~;TSQ;A3dGX8`t72y<0MYr1IZd0qvE%{vl6C`>G$H;~?y%64NA7fU z#jAn^=v9sFpUqe2n-sb!cXz?tJHoTR-rnBEV+%2D(;O2MxA?c-&r1oAr>)hhOD2gP z#t|1$V-|D*rt=C5cYVLPZ$+m1pgdL&u2eg@yAK?ek-8K*2L7i7c)*>V?6f-g`!|I% zSYZ}nRs+#$bJp>S>iMty_$>3MD!0fkt`f&bwjjUbnCob+ITmQMI<OtAW=yYbZ-yG~ zIY>`+#r}!y0X3AV=@!-$6~Vvj8iqlZe5FSOp7s%P`BkXXPJ(zUUBVg1u@bzOD3HIE z`{hef6ldI@^?CRq+vWONuEx5O4)Qt^51U`rjXKSb3&&2j9V*pqT-Ej!VLdqDJ)pH8 z*uT1lu`OdoIHhG6p!;`HMs){E4asT{B!YVpma}Cw21`m0ZqRcn+I}hrjZEN$kNzGN zc4OsB%1jMmF`L}3xi_{4hkKqbIQ?^Fv2)FO{t1g~s^6x3tSwUH0_E{}5(u}^@#q?> z>YH8j+$eqfw&mqZqMet%i5&dhtx1UW<Dw2QhN~q-kbGmtYA$S4VMz>*lDT4HJUlI* z-%C-?*KH8T^VHuH*B@x?0L7GZa-mJ!hL?AB@Ep~dSIv)}zn$^@>XijmRP%o1@@uFB z8oMcbI5>YOW`PPrB&r2~B902U(2z=T=(nkWO~9_y&8EvqI6Eq)!JLjzjnm85CGLQ< zOyRpfv=oxip|oeHdi{B035eN8f7hcbSYlm^f?DeW-PeQ{aO-q%-t6GCcD+Wrr%(`- z_(nKb>@6-Ik#59UF*f%01%fsDxNqb?Pkw~m_qm;oF3MkkD)dcW{>A=Q<u{1{)S)f9 z*CvwaqV4a`YHNZlX-I*N=c*Y*Y2{u(O-m^FSGQUO>%?u9{6TYzi*sB%17#~-GqHlU zzZ?h#ZPp)LDFWH6-B9%Sew`sUD_8(z6BTgrk6on^`u2%_)5(d6AjAAjOYW6{lcxE1 za4mzhezj~7^37od<-SsXcoDq3yv3z{Z-?{Ag5xkuGl8m$fnS*<eMwLH)o%gyvU9RF zADCb~w>3}f=$sSn)^UUGtOoCxRwV|H6ny@oUZL%+3W`V~TyejP=?7`6m$s+ypLF9Y z?ClU%FR}i`rB>$(>6Tt8ErFM2v~o_L_}M&Cdg;>iL<h&YC|fZ@vX({;T#CJ6rub;b z44pW~Bm44MBD!B}&)U&dhAlP8Fy1Y69Xl35n1dG4ZPOSGrohN#zw2gRUfy}+-Ii(B z;NW234qQEolWYU_tgfgL9s3n!a?aOMNjwSano1HTm$P@EO6K9YM@w{KapVjlzYi(+ zgf4^2CrsC8a5J|y{=(8{b^L)7%G<}Wd@Os!`ji80#DUGGgPx_wh#56;8Cmskqn-RY zF1g~=%L`A+0-$J;7b;;a%$?PbO+H_}EtC7z6In9c$;|wg(YOB)-DXjCz)Yml+o$~< z$|H`~BLe*@><m`2;m6k1k(NrkEiI?tC7(ZdBEyqN<aZWKR}m2rmWk(Ja9dVxHa_-? zu9uJA%2tL=-f^RQrJ}hVO*&OkRYp#E-q#_zGR$}|I<R?UrmA3FM_~<3Z9yqObdTfy ztfxtU6+oXnf!UchwvNm9QtCZO{H_PJ+hfSyOJZest`iYGf|=Gp?kQxhnVHf3)&n6) z7w7dXB0uq<DP635RxeM4HJp~VkDaBZ|FfW>Yqlw!&~DaI+b6c?Vr)hr>)R$SvI#{S zkX3RFYsAU?5IB%IhFw^cgv-hZvW1a6#r7t8u9igK1Y(`}6NV{Sv5+X<*LU&9^Mxw0 zsAELG?(@WIOTK~z=2isso!~kFZyFMLUtFkQN1W(AGcx4P&u`aZ#=Pg#BiLpSZ4v~1 z?enS}6IC1pe=qA{=wguVD+e6jpn*Gm19BHzYadn`&EP5i0MP0AG(|VJOz$~+bk?^2 z^Ne1bmRJbo&m=zi4e-@WlftG^)$DjPk$3NH3|XV&l9HIt68Wi4xmLn<E2s(<z@uPr zM~b5iMwMSk;%@|WEJFU9ac>gw%lE#ec%?dDD#<Sr5z0(tukJB>Jh3f$9T%=nwH`T} zUOb%Vb<Ho~TH>Qw%(KV?=JSvV`Q^*Y1Q!?_$PZ6`yMP#PiW~sOXDnJ3hM`gioRX5# z{0~FpK>(<JnGyTdhDJc_LU`9Ku5MOqe=%11>%>EgI7^Yl0ltoM2lN2Zacmv3vRa_o z?R8w{75|79WFI7tXEUz9J4;yqroPTR&Qc<|&`6y%8dR?JMBsmXU<YGhNXm7PK7@1e zi^rK_v?F=QULB9RaNK7}Wp`Q4=oQKR9G<pmo-|ks-n+o0P>=P}PuQ5>C$wr7@R#lP zWL|$NeL2P_Aixyn^Cl!@!+{ZH9HR(6TDRuS5{I%AEvg)xoQ}EE7lUPnUm7BF4Pwz2 zWrAxdW|xakXBtQ!MwO4W?guHWvc%>LtT@5i0+3Z~rmU(eJM&dLmW+&SNjy8d;OhP! zKgokxpal++h1HD(Dg}pn-=$H)f$TmjSlDy5!F`*VVfsr$!yk$2lj=*4!~uR1(4gs~ zFTvl4oosAw$9EK)D*?W9#6<5j|NLnV4p~C3_+|SL*@j~FNCXtOM-)3@Gk)TeM1P6s z^)MJwA|n3zd+wH(WtpP7!FXC~X6@z{#X|V((d&WQ1H*g&{fB7r%D;9;!e5H4{-wOv zBktddP|SEupuA7TTNp@7OP4!b2om`BZNg{klm68)68=N<<m>;P-1=!;Im%d#De5KM zC`82VDC0pCB&iiF+<yqr>Hqlk2VMWa$Kj3i@fj~>|9cWMLaHD1LHey;3er`=e%^<@ zZJP!}>wj;jsNbyof7JbTRFvx<J_?T=C?F^yB_%2)-HeKeNGT|tqI5_%V<1?7bc0BV z2uKb+2uesv4n1^t$1rm~qx<_?=UwZ4*LwdtYaQ0w?!BF1=6UYtzCU?=uB#i)8R^Bp zKfOZal%(%P+#ue)9QNNB?io=M0K4n|g{zSY?2s^y3HNioS06^{KtSeJ`|*GG`&?E~ zOiYW1<}Whg_}?mG577Gm@8$oN<9{3JjR{>@4mD{gnY0>bYcG*6()SMza*&V<|9*O@ z>HhEg>eas`|NGh&%-@s#1NQ2@wERAvzk~V4dN2*jK@~Jk`A(Ga(!W`1!e|-aO?YE< zwVsiYn*sLk<=*-Kc<TS>N#5gpN&3ha=CNjJ>H=q8{r|gFef}R?lwOe>0Jvq%dkV&I zT)ce&hOZz0Z&6$u@pwEmf7wDH|AV?npG{=+2FKT>V@K#I+t$mf+|)l!dt_ruxke$8 zh`pJuIneI2u=m1Z{EJIvO?i*t_WgtxRsSaFVG+tr?ChXEw(I2rfQnN}YM+4(mXlue zqd-hjk{YRZ{aJ686YD}vTc~b!En@DGJJeEafkA-_ipAWoK5sd|kmUQof=xl|qS%VQ z;zgRKC~;61UXF;I6%Se%9~)!C@_mx~@2LAg7hE9p2;JSBd2sM50P+0t@EU|&5L|W6 z4&^Gsg+K#&;Rn~(a()=FE$Dmw`nADu&a{UQfitZq8wBmX`=>qCp+XPkK1#m-j4`0; z66N=jlIXh};!^!i|G_Nw>OZxH9J>+tuwi-~oEBQd!2rf{A<zC?rl1qaNsNw(>6=u6 z0YxMdhYsXQY6Bgi+;89ftV;?DIUFDyEZ65&JPk~0H>%F|PWNE3G3oiX|J?wcFeoJQ z@bnfNWIhChD&}V70i5+$g;*=W97vHASw~-6nVFlHi=PKYJ%Dx9J3w<*^}}}u;KZ`y zTY(-UPsF`SnHyi%?W!*tCfycUQ?4h!dj7PAhK9e%hWRG+ky(0vJH$WE>s=>1_x#0w zmhO)NG<jB5t~<<p^pr-1&X>VsPn@PFb-Sgnx3vQtwMWa!629^o>ktzq$oDWN1O^5s zym>Y4kx*}<AmDt&3+9<X-N@nY(I<Ki+)+*Ng&dC$amFY^$Y0eGzAf>;QznQg@Y}?c zmP*hsj+QOe)YP>88eFfCiK<%(u6t%6c3Ghp4V~HZr{_@6Mz^1q3N<64^GjRW+LVBM zUuF<*GhnEz>j;+DzyJ#f75Is)99_E#qifwDEW@nLzTD3D!4<%t+C0CL%*@PceBLKc zLy(dAA>C3v%<R=7U`Xi?L(2s`Wqa7#58JdGKtHdo{ftq;R#=e!`_PwSw-Kl+rd@8i zb@9swzxH)$`nW6gYzeBs5?tptbe!BsG9j|@csFa<7b}t~qNb)M-Xs|L=Bszid%^p@ z3>~wDIrEc_uvZ`FMYl~$Q55xGEs6_SbhWa&CO!Ayn9;$bG02D{9lE%!3|KA{!eFJD zisayVxTJxlm(cov3PR}l!Ipq)H371N7aw^){+#?$R8*A4ckl-bcd6B{!n#)O4>v=V zFVnXSR;j=l0~4zK&<+?_1Bi3W(Xhj$9vlL^xKK{J#X`Sf*3ld2(+?$Jq@he(t*Bqc z?Y-?CazD%1N~=JDOs3lF_+^+zZ3)$eCYH4l^i5m#Qk+T$_x3z@Do&CmU4=eLK5^*! zhlb93Og%shGP7?dqba~}cENZ4K_JqI4OdZBHEd&84p<2a%TIy;>B|4M!q;%A-7J9L zQ7^w~?qlXqF#VlbBp-p|4)b>OHF@>0gnLIH<|pg0uKCjh%i~3ECY&$T=fi7Vka<M? zs2j>a?wo7S{kQY(UjayJgWtPr1@d$5q^=vQ$wbi3IHkus+wDCXCz(^rukW3)_h2H^ zs;Ug*^!^6;3U*UFIv5|GtUG?v^nd2K!D&+ZXIToI8!#*e*y|sdVwvyx4^diK#9n^o zi_d0@QNE{@kKxj4iM+}ZvMK=anDNPhVwh8d53J?ojku?nm(Ln=piH9*J3DCn%kV^A zo<Qc?i>#@9A}_p^^Ryq*pX(HUDjeE%yVycUvmhDy>+yr*_(r>8SsT8%maic%Ll|H{ zfp8j4)h+6t7<DggLVbZE(-!o=z+kWjD%Svmd5c+GH!^>D7?b@(aF*D$vEeLko0+34 z_=I=r(mSwIuq<{A>b_obZ=4NalfVg$O)oNdL)wfbeZ@=`S_GXtFq`52<JxYP3Ee6# z?nTe%96Xf3LYAXC@70BWVgGCyd$gc3`9-O|or6Pr@gl%hgihw`;cgPRU)d<-aOBg% zysdt0SN8oYcMI7uG*8{*X)YSc3J9FDtDbuTI#<7@ST962y|#>Xl#~k60A@!J!f;7< zV!Z3<n@J{K9Ds-X1tJ;L-Fi-knfv$#=>E3=IE9Hx;{vmN9vNL}_d<M!r=(-nFU{l* zX6OOG7$vtdzFn$M?tL$i?3DM{c8f1ngq16SJJDapnVC}oVA6FmxY}?>>bGfKj4gdU zbTxLq#qP9T|FCl7V{u!N89Vx6VJ+*UzBf1S9@I7*e$|XmI!)nW`7}3fF`;6K#r?ca ze$G5(&)MieGdw$bB4Wqk?!d*;G;#&^iI1scvA=G1|27I(X~<sLrNRW4<peNb`i%^* zwTs{Bi4DAcw+n<U3bNSIU7H+9Z6~%KF!>NKh?^NlezZCvni{K95}3Ju<-T!2?uE^R z3%za^@}3AbR<_&-HN$zv1>odUw@J%^=(SH>4i*2T22k{-U4Y!Ibb0T38yY^4)Aj?r zW!F{w)!EpnJBEBw=tuPQaw(R&?q6<S4g)@Ub(76v1TkGNRnskOxBhSv@A=cATF(r$ zG>`X7<jmhIVUq(e3g@L38b`9_aud31>~z{cT3rkp&)&ZC4USISBCwK{RV%;EFKxJH zs|20-_LGIpqIXaKL8>a<byNzR`{*)+&v=eEwX*tJK5Yu*HNphQ7-O!{TXYyU`kUM} z4U%{zX|5Cbq3`$f`U}OY2Z1d_F4*WO_PC#8W1ofHi6xTV&xJq}CJ-{2oiNTD<WE5u z$<N?I_trIsYJ};(aJ<?#xoo08njkM*o}d5O%)En78E{b|7c^p<Ul!%(L&F)BE(^39 z>FI3KPeV5+4#MmdF{lC+fH431B`+eRULa!0)1+%)aJE8RM5KF+<D)uvpN|1H?Q!Fi z4opvFzu%l=LQ)dNQFD;PsK8RCrKNEa4R@5tF(`<*0nX<D9EiqS-z_DA3Q9_53_p!g zqeh<a+YD%W2|$Ypx%zNNyZ*K%MZ1d(R1%>NVQAd=>-XpuCg<{LvJYzyl3*Buo12?z zjz-NRCJ%%IVFsRFd-f>WR?m!?jFv$shDI#5ZJJ#ywrTR+ug}vS@EL)?SjIy$zRi}@ z))WLD-ETTQtnXy)iCgMV9nU_rzpr&FqT-Cbb@|)IhFfd{cO(-uuYN2BnVc9UA?QMi zON=%*f8bb;jEuz79*S6h##o>KO*earqI{yPO!mjO-y441S4)#3A|i^>(Tz{QTS0VW zI{-7ui!IOS)N;-MIUyf9Jx5efDY-wyS(4XQN?IuSB61pK@HBQ@jV4p=eK~!fALRJW z>Q2>Fbt59AjrupT-R63j+GefCd3!4)vm=&wiy3zSy}KjJKvdHrOQka!y-h~EEf%tD z^VF;YAZwkL<&&3d4xC@Cv1^TmR73wcV7_zVz+ida!bi6zdg`3)Gs5mX+c>qvq?U4c ztOznsE1mNLtwf{V#BG!8D4N}yj~W|_X~ka>{F4no>>3Nr&7EIo7Wr_MvT{jP=}P9( zO4?OMSK`}>$rvh~M$Ii<1NHvI$nSll7Bp>UHJwzaMRbfV!3b;Q{l#w>ov(p#0ZK>6 zluyyg(!>o#!yCK21vtg1qa*M8uq$~(YVG<|w+0|mS2?8LTe>UYmmjd4S*Y@qp){A! z8OudxC-Gdnw2E~?_<k6i)5y>oLivuoSZ7E~w~UfSEzBy^Q+!Wnl=Qx{`E%;`t7%OP zrj*6ib!X+{YDMxYyFTP@Gg1s0YTvSRedgOwh^R<;TkLr3&4f%>`F%I0RaR5mpY{yx z^K&+^ACGaM&*30OuF~Z9f`0X=&a19FE(T&M8pk*89)n??)R#|WJm?jT>0oR0xEQo( z(dd|L`qIC-G~Xr`cc}?^D{Ipu4K^EU^P-5LT%h{e^6P#-yyE+d9AdJvOGG13W6TWy zfPGMV?1}b=qZ}W*LTD!s9C;w){GldeV{B?_D(GH=ye3b6`)?E5CUQ&T1lSIG#!CTG z9}(-zMel%}xJVaR?Vz(lcjrC(%GOb=3sb$0fdNGQcQ#)uI5#F0CNV0tOLTvHJeFBZ zC$W?fI4Y&CiVp%bL~S%|U01triTww>)ARGsn!~QKRt-V<iQF7-?JB;(w^;yj1-%6* zlt?lLoM%3FWuMO+4MXxantVfZl2<csVC%Ezqy|rA&U=CuskN?X+-7cH&o5CP(;;2^ zy0v3mVF`N*)L%{XUWlU|^Fd2#47~`5W@uylg@QPS1n-U^F3s#N@=r@Iw+DM$jOWVw z5*ET<Aep>>T-eha-dZs7IPsvy)dw%6HoQBVu_RShigWe0*=Xc864)pgV-~)nV?>E~ zJ}fmXPB1Ai+?sJ~jhCH7`Dml29*oltn-2?13f>F<F845&_0#<;h3)4uX*wt;9X;&q zR6C(VhPCkpI$pim3c#v_H2F=ed&KO1y?+Ioy<b$mY{NWKksk!4p^GP%^DmlQ5>QN2 z*VAmw=K-A$*`%h)!%}p2CE$(W+U|#vpU=)(UIkUbuOW<7hhhSF|7>0zDj&VlSDgK> z-)Gb^rHk*bFx&nWu)_eOV-auEy!W|_crK#v>%(h=vx@8M()M<CIps`0D<vP%($KJJ z=$OpO47F_B&0Br?72}#XvT83Zn{r)Rx}tMcs~p_=!vI1nxX{hYl`rVn`0w|xD66Ox z4PAo#ZIT5FxocninkBD99ggtS(bo^x>e|!u5;z9cExxF*$=|LPj<=ffi<Rw;@rhXV zs6^cC@^(V*)uXZ~M9Y7?>AS}f6aJ`55dRdw5{*x^pOL+}9bnJ0OJEt^+s5EKHr0H2 z;z@dW&2B8d3&E&mz537$O@qg;X?U5C`=6r-I|c>QTP9{oorT>Zi>+$5gasuvTs(3y zp*I!S_x^BO&s*E)R!rXE-hZ=y;}U3|_WN)D)JpIgR`=aJu~hnWeB)HjYa3DS($9SA zGS3fX2OGW%X`1Bt#Wv4XWDs;QIim5&VW&ZQe^w!jW7l|=qjWwTR_b%h%GKbl*0>h& zWJY`BER(BDmWVyffE%;sKukuL@M*UU54qayM&TA;hO&oO&ZpjMiH$ikway|gVyq-T z%#!9U%W^jv_WRp7<UOy4dD_Zp>OkoluOJu`-kEHeyBu)t%t2T~h<xkVZ1eT=l1`lT zI9G|em!GH@)+j`b?;21Ypw8;w*`h3}(dUaAc$<vALv~ogRHRGWCoW5%NFc>tukZeQ zBS5;dL!ylVun{nVlM?Gp=Z!dUh7YfG542Bv(KcpRJ!@Sju==<Z6h`7SxGEKfjR`Zg zENm9N9ZACuQ$_ZY<z;s`3XumhyHQnp;pMK0_n~yQA0!rpIcUSJyxFggT*i-Kkv=U7 zIfi=ETTd|qZ+9mjOCFSmQbZ>tL@a&hWuXer`~LkB@s0Nly~;0CE`ruc>6Ww3o&i%= zoEQ~aBGY>^X8HQ8BPz-6&+>J1fR%Xsxho(wxMRJ7MlTli_|E3Jx4<hO@<5#l$J9F9 zvrcW4A5oSwy}o-VC9kTAZ~hGQ+Zybi65pWtdTzI-?~DU^fty+-4Zn|+n4i*XVZbgI z!%TR2e(6pvT1yhhvJ2xA1<5lP9XY&s`_8+ZyT54U?<x^A?vbcGR2snNA9D6a(PzG( z@<%P&j`FKdzN!(;Cn>*-WX)R{+w>KyM)m}BU-W-)Eg~{13X8=8s2KvObwt)`Io);8 zLPjvF&0a|7Tp$as*u^FI;P<mJ#J5NcfWoH<{q~fS!s{&dNS}Gz2}eTrT<liASOQ_n z$S9qKne8G1u@HL#wecsgN0ixQcqOsqN7<5TjGBNvYZ<rM4K*!h_o%+lPc}i@dtiQ9 zQ^o?oJsjN!$)0|0b|s;6n&clc=h*tJ4Y2HPI*a3Tb5@`39h6pu&zBY3X7xh_ILqMv z`<cQHeNLX7`Mndqvt`T@8v5e*yh48WACJ_2&-yBW#9J?w)aI6T*)f{K<IlIt1i8U% z>u`OU$Ng`AFDwnd)Dt`||5U{_7CH(ygZt)XgkxDDoIyWVR_!%pg$y4cJGw=&VfT+I z5Eea#$HvB{rqVqf)KF5iry*S#DYMJdUif_~O_j3Vp}zs2DR(JM?svg4LXqeJLw{+Q z{1UfvCzLB7-R&sEhjAYM1HF`2-8e#v?bG6)#BEyC+6S$wxwVw>qm<_`wmq$3eecAv zJ%#8eYN*Mjrjt-8E;qIHSxi@}^eRv4UrF4-S&I!v9Pk_(L_?heThA#x&o6W)MNfjN zd$)J106q6=wLtf^5dF}cVQ|Uo;Zc<@J}}DboBQZl7jloc^yz>r>3Vywc1J==-&M`O zSfg&0ecO^>{ZxfnOk4x=6UC#yg_cA5Kv9a5E7f1iPO%L7dr&&Ch!oP$sk1ARYYaM} zxp(U0(Py-|x-C|ZA#y%zMUz*ovD0$PfQROQ`|#d)ML_!FPM1pc%!ZZN;(FG`LU$0) z;^l4J<XEw}DVb2rqZGIfB_FA%g+ZqL<_OyX2qw8>7>dMonvY4CRf@%oLM0T~3%R(` zJ~nXvfMUZp`kw5tw7xh%AL={7fx2sVmRWIiX_LBC3j&~e)$B5^s@~S+t#$Jx{3tI* z1#xZ`PPz&u+mM_0dTX1sBGh1enNW{$RiiTPbthOHSaufHGER<Ei2!U>`_UJtbW4A! z%mZpRmj~00*^4<&LI{mMn^Lp47gPQuG1i0UvN{i21WIA+BGrd=hIvy8oeH`a4|nI% z_9a3!kKIUX8Z3^HXdEOkkx%7aWfUSUW78j|>+>m?X{KWJNC9<c_EJMk41K1FXq+w0 zq0DC0wwqj&UWwC2n=Nhm-)^%dU-S;pGP__U!k-0dp;)ZZprzGutw!=<H-YeZm_-ls z%Tbp9(m;f9t%feG`Mqn_+L;dzzYlcry)*%-5St^Qm;VmQ$k*NEee&f__?`6duAK@q zEQ^HB<;$1ZicHSM{|^grVrcJQzwb8NxtL!@Mn+L6=&d+JiMxk_xG#!)K;AK{-gkw7 zv)V5(uiF3ahNBX=U0sh{=a{$owG$Sc$PVnH3onZ{i*hBm3lCP+yen^=X*Yl)p`Q7r z>PV+B7!leNH+D2M79%c2w7XRPa<O~Jg{`ZrBR{4xdBoMCO6rj>C`;h1i+Z}qc?-Id zApqD_kWGqutq!S3rJ)gExY;$b`a}D5cEO3*2*9bnm_pN<+1%D9HXu@T8~AO=IBrR| zLAA4m>Xu`oWDs@VtbXRhRKI7S`8YtR9s1>T^Ilkl%VCB`=K$C-UrID%<^79d936@h zA1EG*$k9Gklx`UVXy}&TvF~B*L7PWaS-B*KA*>?g^v1wUj<+msRP4R5mZGEIi8;j^ zU4p&?3B>ymJ&iTS2Ih2kXxlEUbVA7`%>&wZ4NC96tzd0Z!!KrVPAB}X@mHhqkc0NQ z76Y-7Ee;6%n=Z4~ZVIcAftp27|1p2tLx(ufp0izFz2;unJFD01Wf0VA-aKy`T0ejD z&1$t=f5&RU&qGHn#}~Glr8Y;lRfkbinP)#A`ZP^gvtOE-7)N?`$qbhyU0+fI<ny1? zH2HJ;0P38YGIJBs>wm4U)SPOi7fSJ4ojV(2G|(tw*=6IWC+ot@ZnpFCwATD&R!VUb z_v+7u*F3GMmby@3A@6Kedr{r`H3BV3pN5_8EC*Q4mz&Zsc?F@BHV;&ip{>+3`FLtf zL(zH{2F*7h67{}cr`7mDZM3BknVykRd*pJLwi@V2golR50OWqEF^OmSojzYi{>2we zA>_{VHf3aL+gaJr1_Ef;qcHd*#!0QS&@NEux{5|%e;yBa#Jxruvez%CJ<`;#PTk_A zKiRep1$4F$I(quq0F!Tm5++=nJL?{!Py9ceeS|waaKNTvdbDXWzYs4=&kU(;S#&Av zHp5&cYB-6Gx1K_0S5!(+a15Co>oaH+6FnMwE$Zs4!mh?KjAo%@esp@KDoWxQOnbr2 z#GlKKdR2MX4u9PEyssL}rlheDhj1>sHDWgKWOA8t94A+o_8HbtbXRav9w^&QB)(5B zcelyD&3i0ldw6JPhyulZU`|mvl~@#4<reB|f{<BHP+(d5?uQ|4MV%TaaQMi1Wwl;o z#VX?BJrS5+C+1W-AIj9&PVnl|ALh)$SX7<LFVh2#RKYWV|EJm3ZTV9qRZIR*?QyfA z-w9op8Yq3(eI3v0e>DHV%9Up?1|!t(GUQmw(|PLmpLYN=vEiS{4W)oFL=d)WdfJpf zQIgs@3v*t5D_wWbDXRVM(7%^_ABKX|S#-|crIA$E8+?22Ap~l=nM@(YFdk5z7#fVE znHzj4@?icmw4ie)9J=#93Iq}4c=j2PbOKRLnPMe51AAeIq<CT6h>x|$6W^_+I8n{B zi50DKz;neG;thQ7U-G}_<T^jMh)D&d!KVDKh?|6qPPa1sH1W?AiH&1O+isqsH1*71 zVZZWuEZZDAE&vSmk9GN^aa(4$6+kUetv@R#0e+DjW>lVAOVyg>v$pZo%cA*GG$l8M z%w)o)GkWr~H{n50!h;~<qzNG`gmsD<l;;}0)ZPoVZ@X-dn)`!^xyXIiXwSRmm~oU$ z{@#rMPW-cjeZO-DSHFKQpZ2hd&0lP#oa8Ea0c^Dzj`FST=e3Juqna9b{AXup7s+Su z+ZO}k(ffW`PSa%&msTkpM(5n|fo^l1GLNW`kOp6rOdCwTmF3u%CVqSu+Pu>>6!VRZ z9U{Y7mkt*Lv+%FoR&bYLtvWHGs`7U#Wu>I%gF|Blv*e+u<G`pGx9JMl?#!z(szQSI z{7r-}0%b4MIiB{2fJ8DE7w2L~+jRG*u$2?Q%FFXHh3Y!=tyNThE!ZdD2Ywc4U5Co3 z^Y);80b_7g)8!4+?x6+LVAXJmBfG%na~I%<NM83203z@~KNUpoc;AKNC7WcVt$m-K zz4m%tef<%gE8TTZa7V~)7x~V>i)rc)<c#$Oqtr8ng@g(q$i?0viacxjZX;%Abyt&H zOPAh`uH-QqaACisuR~#pUBtfU+QUcexxJy{g|>QBWdu1I;t$PXN*%qvpOMKQ{la2{ z^ES0b-WTYr<F))XRj(vh6&HUi{&`!KMx?5^G7JKDz+M(>lLJh}f6x0t&ySv+BDW-b ze<c&m+RJhpP<4(s9y9VaV?plsiwR5*)8t5tZbX#xvy=DE6ZWayyf+n+ELK}`_h!+c z;=RfD^0+b9CSG>642geDPUey9u#vN1erifK{4PgoKo;(rCJ#-wQzt;5J7<BJr@Q-6 zqohS_WaE>Xf(HPVwWN*Q>35{gnFl}cg)iG#(w43cW?s@Qp*9vY4)9~tW4WHA>IN@n zopmK=04(2i{XLwzZ?ybSKu5b=4hu0XhKd!aCtC=!;nEEncrz!fQZ>QF#D|BL);xR? z+BgA2G<9|U0A->q^UUB`q~=4Jn#wVwwbXD5cK|P+NtxO0R@RF>N6*=m4-mjP+jt=8 zi@A^{drf7mEa&%G*e0X_i>C!TDxDA-S;l@PSUtR1!kZZkHv&<{tn;ON0+|Y%Iu~-F zfd#-BkCEq9q+T;$6hv}HMj0rU>XDG;ndownpU{3+GElUWA<B`}YN=8wpA6N@zEt}9 z&@-Yb8nwR4Z!^z6fkrG`4%MT(OPJ7%mL_9a@*2`~R4Tvjn$gGM4lRxA0}=CO5Z?-n zzk6J!1O>-&Ki@I*SN^jpma)?w&?MN4V!r^GxN{E)-yU7k1T0E+g$B^|RQK}NJAltH zO+UTx*hrz^3NJr@){{^;mOA0-GKdF_*>?qFc?Tl_0t&;k1=)bH2+NF(jy@?C`poID zD-IgCU0n|gLL}eb2CHZQc?<UK1w?hQ2-@@FL2YdUt>a1!wbS!-OVbou>Z8UZG9`q} zMKh9Q`WyvvwKwy`Tju9O0^T#%Z&oAX5)w8u-Ya)DPp^}JbY|CJI)4*N@fXwXrHvUm zDwcm=P!I=%dmVu9CiWrS(iOfEnmUNzhXxf4Rv0S)Iyt5N;~{MVHchkTvT5_A<BNBV zx`LyX4U>+4Q`Td@0aEdT_K^(4MKI0sP0fv36q->N&!WPg;KPvj%5oH#MKaYT6UZxY z!Y#Lwjbqb^Gk2fCaL`ko5L?(5jn{66JUHF<bsBJi5C!+^7fc;pxJ>T__1M^_8}EO8 zu3#1_x{5HyJ)Y3ym+{5cv+S#(A*_&((%%auPP4q>AZ+%m{G4hd)Le}mdbd67X5_55 zO6gF|>@t-ZD1})sMoI3Ei^}eQP)c;IeNQA4IQm^G9l2J<A~WZzUNetc?bpj8AVO{v z@3W($C66P@Zna2S(D^^9;w%6a-PjI|%U_3jVqYSjA3P7yt90IZq~)qz^Mm{HNp}23 zt(-#enR5Z|_U=sM1K$IDrhm#T{#CEfC_wr?JLvpLS-o^OsPbDamEply*2!@K1PA~X z257{Y3aIflO_JFAvH38xEFwVq328BVWCk27ATx9Q3#0@2eKz>!I*Tci$B;9v`$4Pk zFjY>0HmDf|rUP)@Zu(xUKJ{(q0?g{}6g8SgQJz>ZFhVV~s>%}n>`e!M&{v{6wD1zE zzgp&1)$V%>>R&y1KLu(^kYMY9?pjJp>B=8gtP8qZ?+M*;pP6btFqP2S`SmzYGs`0} z!%$4c$<a~pdKwo@paR!XmYL&;4uE;cY%MULycWg!<0tmUnl{Zb2cD1LJfF`k-RK!; z&CWUZ)&k=tgq2w^2~P6;zG$L17C^+m5>m5etZ<JuKHO_rPnGLF+S{(6$zriQzATIa zjfj~)H8n4rIr`X|@@I4pyC#{?k&JdG^j&s&TW=$(BT4Dd1~(VKRR5G8FLGU>x$A|6 zTR45pd0U%@<%C$Jw@0>e0)=qP6(8$WW$tnuS_neM6Gnc>MHQ|7Fgeh{D(fpCwzI)0 zX%-%S?gP5J$L}BR)jCrP4(H4R+qS<zp_y!_#BGXPu_Vb^+j$LkY~{KKXgb`cXz?D| z>cpT1JjO=EM86JsZL^w&r^;@9-)AX5hzmv~*WNN*uh_mW#QNM|J!VqP=X)fsN5Oy7 z+=pkqu|4F$=K$Xn|MN3tsw_W;C7<w)H2@N5;ev8o_lf!&SfiE3em7jKoeW0aj7+a= zGF>kAj_<q?XdM7vFDIsC>1QT2%l_ITZxx6=?9s<ikkdoCfA8o)h0o}p=+)2s{ZW{; zB0ZMxN_+>iI3l&6p2aU-F{Uz8LN%R%T&(M49;+aLWHx@Etsg$f+a8qphO};St9rI@ zC59JNp30Vr?+lO{2wox69y#+=0b75%2l&ZnJjZHT4w7VX!oOVFIirH~w_B4AFqmBp zdw2HHTkmr)QypH*vKxdr{(kh;9PZ$=uqd2HrTc<^zw1xPH1qD-Al6QqHZHBiP~3In zRm^V#%$|;*-XCFT$*QQS{e%1fJ8tA%4x53?=WUI3YY2`O&3C<bQlRRa#G<`BjnR`p zjrpZpo=`|a^|h}y1qV^H6#k&AoQaGhA94&UUK^3v-jAb@-stk}ogb~01B0l-EcJ<n z`SV$*C;nZ$*thdNJy>SXjpP8-#;0`4HgV@25B4^c$5`)5a_v9iqCae*dOjWxoqcwG zH;!_1ojO)#fp2)R%HH4gD2ftrNPpx~*n#~uMa#h9LNaNo5QPE%Yfl5+{Xeu2^jZFc za=n<)BuZA!CF&S-3onwo4u7iJ&o1@v1_rJVzioMe(>Pf9)4R@Zrg{q=V`tqgHPk)1 zRP7mMg{370f5-csy#Ajs#K>P^2tb$YHA%F2;i=scDz-842K%VCBp>xk-_^;zr;SmB zxOv~#rKj8d)5gu0@e8GGD2Gm|Jd7*;x2YV~$ZIvr(6^ed@{JiF&)TxHb01at)*s*q zxeUX^6%1i;nqzqDeT<Q?p#Z5K{`cPWF8+mI`HZn3iKPl;V*5_ICZrwKwwsMBwQ0Vj zZ}Ga^m0(aRw9SOWX6*KU*&22;zz@4IrOFs128q3kIBz@|#5l3jmD3FkMMtTt?fS)i z9#{De&|(O)(c5R{+bXU}$V8RJJY$V&;-q>D;2V(ILOefITv+%-M^iS*eD0;__1I*i zZ-gy-!pgz&`!*ZPSl2b{ZPDRa+1={X=V~KqFwR>~b7d>rg!E$5vvVxPaBE$01sPkl z-XCDACqwpbE$iswO{z7YU4E-J;8rmmhADH{J9=|qc{}etq{M-L6*ImAFoC3U%6Z-v zA_kVg9qh2r*!v{g)fTmcjf?h4Y?#-)o=2;kw_+D`f^(9|WQl1l*owC#wWo4hC|@0) zFN4pg!7Y356O<<}uKh!!(Lw&ZypM99Z_FJkA5zOU!#P@Yb9%m+$VotLFX)U>u2jG3 z&qgP)0(^2IHn5YcIHHdV=~*aXgbAEkG>76`x+T>=NL=l80JG=J`+Yw9t+d-TLvu)B z_|H^Z@hkq4G5I2|H>@?o@iLpc{4h!nYSl`YgAHymepkqwqIq69d1QMqf6~UU#ryht z6d2q~{-nqI{Q4cc&E6Cg{)8=R3kvmHOL}bnC$%MgBbzlp=-DF`C=td;ClbX7U&KAY z?NA3Y6%|+YxRO?qV&LgxlJpAV&27i5s8$a{25PFkgq<<klkA6=@Mjw1#9>S5OSZ-b zBEF;Gh(|R&9Vm7x)fbK9yd*VkKe#WoyTiV8q@H1$#L5pFlCCk^uz#mtONwVSsVe_2 z-MN)rFS_EQ*C&`I)!;f%Ytn5FjGLIqry;n0i6p(M@>(ipLc5%w1M1)C?7v5-6DcAr z{A55h+2RO%p5&$*H&f&@=oV#Y8L*vtI+>hNLG5Rfb;ajBoY@5ur$yu(C#J}$hk0K$ zfKwVjgFeYlx|&NLAihX2*f*ShEnSp%lOx%kA=S3Hcu(drKxMTuAD*tb<!j9a5xB$D zMkWN{&f_O;lVG$6O4sL<3og^r!I$8F3a^AUf?QjJ8K{w4>ZYl)$__b>ou@EP_JPU& z>kXt|pp#Cw?3v$uBB*$!{{}21pQQBQYq^G8*`1l%GxbM-?Q{a|`eUZy%FP>G$hE&Z zZpRZz&|XTg%@o5*QPQYy>$egif4F;#o?$X|xPK>v6^zh6AjD5i=S3%dF9+8HG0OWy zr(-66@sED(@Oy3UlUb5Hqa$QHKZ29MNS3Y9#l-Pe%uPZRi9qaTC%7({qe>xCv8srN z+6(lf1*897uW`^gK#;L6r{o!+$lD<l6yz42(1F8iLI5IW_{)RI+~#3i*&Njvn92tG z#qXqWLseGx8*=_Dch2Px`x>SChT7Vo8A8Ge!)Kj723X-_>WX^FMpXmvQo;h-NbStI zl^|>WZVGDu^kH9KtPG+W!v+s2h|_SjCNISng?DKb7WjKFw?BuW>k|*Q<_DN7Hk*yg zLlA7rNNMmyF|Pdmy<c3LR004XTpf(>Y27V)m{UnXKHug!-GoyC?+*YD<;(x!Qgi0h zE$7Nquk>GsaX-cLp7R@D_lMBwB-g)uf;bOhII=8oCOO7!N_A+{vUMp5Uo8I3XFQjJ zK80g5FJS2Zi={jHrrAt>suCJ0QiS$kpWQ86pNWy=jG@iMw!okdRM6ytP0u0e;k68o z`=weBUzIn*Y+sYn>P11lH`}z6O~by6ln(bdYWpkQjP99{A^w~hN4qwF<MwHpDAz?1 zf5@rr7c?9ECm$s+GrdR_ij!PfNb1{rZs0g8M63;b8$5XP&-XOU6{9UeoJ75dN+(2y zUdvU^_=ZDZX5ZZOwGJ5EQ4+&N^TEROuO-y(6%crwu%tHkr@tuZ$JcpHP#*Z4VC3p3 z#U0fL?%@Ar7oSc!q2}Gl1L=>R&UV7K;ym|bq_*eJzuq%h#NvF;&D)q5U*~0YortS( zIrS1(Qv;I)Co3B8ehGSdMsc|~0*>H2+dRt>K=fSs)0u9##VtnYXgPX1Hl35|6p1T9 zlH^&d5n%yWVJyiq7C_l_^?!^=&p8SHQIq}dIF_X}-P6;W&<lE~)uW>sn6XE-xn*kz z?k1D{5AE+|HLDhiDQx(dklWRAYT3Rh%#+d}!l3*(f2qY<FJ~d#`)HXRTgWByZ2ZI@ z;Xu45*BymB_WRoaHH9V_D#ZbRVt9LUlNLEW?YmT}5X2dDj_vOgdk?!l1%eMFICA2j z-b2u@p=BWGav=r~Q~ghLdYYThv~{{HX^+um^8Eb;u7BWWUg)=To~-5r7^`G~CcI+& zCng=lro$RJ{@SW4SI=&l>)(VJSj!ru@@8$%5Ssn!X5<;I5jWgGK~_RSk+BS#o^Po$ zMBz$L9b-dHkWMP%jC3u~p#e}DhDlMF^<}+&u)(Us+YuS6o@>7$FY;`h!F1!+ip?w5 z9z_MV_vG0^;DM|hZ}zDdV{v~5@qy9S-pn~$e8V3GL$~LRbcZf6Cp4H8+nzA|TVh<# zO6g=6>f|EK*&`nj_jA9=>?H_gs5QQC^HCjp-R7~%kJ<T}l~eML^6!Sx75ICC_9n3e zAEjWQmEczIZGf7b!4P<n2LsFIJhZwK3B47YTSBP4_Vs;<lcp*C%R}BHOj!MUUe$;a zV%`8|cZXJzXF{aSt1TqN`qbq=%A#?ADkr{sQXGBtvs~JqZBZ^R%~AK9SlOPC{dL@a z<lFf-7a)@YEl-#_{I_?U7|k6V9ALDD>*_#tv%$ew01M&Iwy)`BSyC^#juyEOp}ksm z+3hoWj@VYrMt8=R$A8^u|2`+;>Q<-lDM&OLY!>|>fIw>vuUYN~MK{AZIm0kD8oLHp zar+E_1WZg`{PR2#K^j-i|1Wr;ts(}X3FArVJt0D7nBjP~<(l4`Y4G@cS!%hE5}_RC zZKv&{rR1SwBk<pMFnpV5c3`nsJxD5E-f6K4D*1N+p+H+%zr44z$Av7;4!-L<A3uFg z<~He?w^TB=+tze>%homGKHv@`gyqbv11gp^#BAKvprm2svSfnqku|6DZ&hA^okHha zmLtS!G5{O6H}lc<TME?oKnhuxj{Q-V1KC5jQ^DcaMrY4747&Qx?&yI1K6$C2NbVkS zvlx=&;?*AzI5FA}s)6MMIo52=f0k1Zgo}hPLWDvZyqL>VAns5kO7=}5z6J98MmC;0 zH<gosS?`SdnH=*ko<R6MDRyn>-|LUk>~B-^?&YJF`+G__NO|PAeQ$y{9_2mJ_J;Ma zUFE@<n{OO&t18P!8<F4w;A)%*b(N@c2sC8IWWN`;-go}{yC5-ES?GZ4*ey_l9KLbC z>K5I(XE+BzF};%YPBp5di05};VSMa%m&lb$wnx24nfL$C2?3c6wQW>UEmflAY{GiQ z5uV}?WEIqL--?UUBJ2}L>W_kYQnrhE%g)+0wZXpc!8byl^!FfE02m4D0&;GUzWkr) zA%1uGT8Ll%+9k4kwzivIKM{y6DM7vRk>*(qrpT3<I*R|r8zFjL!xC#{a6<z(&8%s^ z>(fX=tNP?&_>1Ep+zQ%iAVWzSoA4r13UaY}Z<zR-9!5|CSUjk?hmVelp7W#yJdDev zn2gTd-vj`bFx<8G!)taVBqf1%Z6js;Hr4OmD_omBbx@@^is(&}ZUYc3K?}BDyvacf zT+Zid{r|KvO*UR_cXS5fy#qKIy$>OGR;mA1Hz@si8GTMg$s&5|aS>Gymf=};wFMvC z3gE2Rkw7ZYZD;6an+=Hmo4g>^GI)Yt2lU^fu;-uEQZ;mLOk;{znL<<DIa;xLt#zJP z&iT|$#B(3d?X-EoQ69uK_XO>2Q`^se_lsi4z4T$f6Nb+7v^oC83dOeO1v>Q(C*>Jt zjxOhoiW%Xy%=xSIk@&CNZ&?oy@8`VQGpYI0C6&X_K_SG;idZlLQjvBGAU**FF*9og zCZ&qU+9=Hd7S!ZUH_tz%N>c1vpSX@WsQVQ)zv$ySq2;^zC;H}@a}W`E5?g4g1MZT{ z)3EzTaYVH&m-W`(G}lk^#tdS~-$)`>W#~3ydtLE2JtT(DUw~wn3Ej_R;K}e@c%FkK zw$b)S$FJn4@Ov%gVuBHD9s9#BE5A+8sJ>fCLOy#d@Y{s~0iK#?^<9>ct1SadeEHnL znSuVcxlPX^UCw`ZJ?mW7ZV{bMiVnH#mkoTK@#gp3Y`8HljRi^O-ZR8Y&Tm%xekZ3W zoUB!Ihbvae>nEin+{8VK3PF#}Vp+$)-5jvav48$rlUVQ)v`{!9E)rer->*}XV+bw$ z_$+=#<5=>aVqa_JAZyQe({I}0ee1s!^P3Yr==|m{a>Oo<&q!!DM|0Q%Joag9guU{R z@*mRaLkNI9c@O2N_f;0bP#B0lkA=e2DxQK@niy=yUhrCTCLn<1j)QiOn5KZe@87qt znpq|eJpDi_Y6RHf)8deE`|}fRRi9@Amy$CK;A`4icm>SJV$-4KV<W2cZ?Xtl2Gno; z#>FA~IdSqP3)2|`)bKJsaZ3ub9h^iQMB<3uojBhWBSYeByBxp7+C~x!OO<2j6_avO zeFDl+&XYf!lJ?h1BTk5ksV+n@u^<JVH#li4SHCK4)SfXqg7~xzo|TBVMU@s;ZA}I& z?=CN4)e;F@9tRS8Z56_+Yghw^O(+!HJ_V)NmvHOPU!be1mH^Y@0T%_CdAnNWXdrUa zUySO6bNf!nk#Pla+#$q`V2JmIpDhfDF?xxSoD(Dlfvt9=f8Sua8gcI^>B$^v6K=Ve z_L%NCDH!;xJ$-`(*rWb??fM7h%W{k)K8@x0^@@6=12#Aqwb{5#sH|{o@8m=7d`^<` z-o%*Z+$yY>$}3$PDV0Kc*-idv2@E{Q$C{?dX)!Z+){8`4OrdQ*4$wyGdWL%9-hh#= zD`t1RlWQ`<e75Z%sorpF>aDec!kxFEm<#Cd3vzLBakrDz;&okE4#tt<yG34cm7}B* zA7M|WjS%5R$z8alwk+;6Z>#3PwPk;F*kR}o>9CxqJzjedJVa=z$NVv2&Gi`e3WDO} z^+OB!<ALMET_u*C_LM*~d-q3cbK5auA!9GoErZ7Ymi0-Zp+uCI8v{q5C+*RS>8h~7 z!A0J~e$_3B<ZGIbiv#k1>R%rSq$<&z-Fp7^3u~u_-=)tni5$9-hlgHW{m7Q*^)vL^ zVG)_zE$<sPzC1d!4{i~0388rV=EJ;Csa|1hkJety>ZKVc3z|DopSn3z5M^v<h@}YL z?%f)IR&Vo0%)x=IO-pM$+IMeb)+v#=PDEC7Msg@(a6mW&M#?%qe0ccjY9oq}h&x!q zs=l=WR0qHJwcUfHy>?ut^ug>v9?iJ+YJm|xXnA*I0JTsRM0(41#quzgm^n@SQJ&~_ z4^jQPWFPN@^1^KOY}JJ~X!1qvKO?@%wLT~{+fLB^P*hZuoz1%lG4N#1tTV8+(DnaP zysA`kUUn;NxnzAfDam294Zmd1asb4Kl?nW%G?tQ=-zFW5S!1Sm2imOO66cwg@sIho z@xfJ#0xE|P)3ZzKZI9fHw!M<xnClY`D&3|XOS9JoM3~?~hcyi9zC1yo7>bq2zu-Np z(YUybdX;V(I#KGyrk66j+R9I8E=Q&?Ax88uDCCZ5fkN@i?d{<MGL~Hl6{t*9bG+1} zUFbxQ2NQ={BsB4ZT*$4wB=3OZPrkWT%nPyUjq6P%7`;h4`0R>!zP|dRcH5_f_ygYf zOOLqGC$(g@WCUEbAju>JjK-xPWaYKJ1PQW=z=<E(WLrq$FXVnrM(A!&<-QVTXP2Yh z{|xC?+wk!_E$)p@rEYlkMm@wcRo~=oq4n&Ym}%reu-9_ObGWk)oFy(l68FV0!rNs= z_(5~zY60;!gWv?yGF3&)l+1E@l(d_g6rx&Zzj}0k=`zuDsT(;+92uHU>V(n};se!S z*<ClJv-MJ8_J{_NfU~^g?>)oLCRXp>wvWsLd6sVn{U}0&hTUkC@48RyDxokhFR!R5 z4OX>e&~B*g5j@v}s`Thux!94z2cx;Dop8M!((>$bcw9-sMjvf-+HNj(`DX5`)OQCV z?u20?PFbQ?-r_nwY-bqe(#nUpg2JjJuw;Wzcv~7U#pX26=CjlnoaoD*gqiP}CVEUE zeGS<e=eHD!4k6-8+oXxJa5dC!$Q*hwD|DPr-0QMV`&*`1HqvfdbKeTmcbwA7>dw(G z{rvYT-~D==(tf|tSoWn}ABwD-<H-%eJ7;CY$e4`Brea|AAdr)Zz!{1HnpYm0_x&ca zdq4O)Qg9nzkJ(IIz!q|3TC45dXIcJD-H$-D63E1e9q|3D3Pzv*et#MZo~?2wj_~s; zdSxXFRn*LTd@@P9)GU6(fiTudVRaNCZv8sS=Kx+_#O>&|xT8SL1*pUS?gUOnP_YW- zXg}{n3D02{JLo<#s-Rl%x}Lb;MtTxBqpeceI%~shHg33ed!m1*f4U))PnKB)GCMol z*4EY%HYXn%*wOi-0_S6cRaeMdC;2N^g|jdlxm7n|AM6_v-`t7rk;Ro$T^&A&FkhC! zthOQddZU^v_s|BY{+|Y<U62;A{jzc)fH3tcxJ^fEWp3M-2<vIv@NuT`1feI+r;uwe z1TFjZTtkimi+**uL$Ivzu*8V8zQ@Ab<?>arDMOzEyr?Y%X4TrB)%?A3yZ9#JXe%SP z?2bDji#Qx?b)ybGK*nk+JI{Boo+Z3kKHb2>{ZP^(7qi0)L~CPQqNmg9{Pb1co%Chz zxWepx5WjqY9_WS$LEo<1YIC^K3#jKbns0I~|7yZa)kP6T^wbJ}zY3l<+8qu`Hyu;l z{P{Z<pH=@8?T<e|BUd6>J{HRtSB|Y}B{`A>OYi*A@@^EfR2&(^ce$af)}Fqm&R*N= zps1M56tk9u6}WtEQ1>)HaTM+S3%9#^f;rLhYpR0LZeXsry~52~>F*Q0Z+Kh(Q6uiW zLT(k;7aq~so_VEZNEpkNY@q>71TD)uBhm)28e4TIsuuS;h4_eYBLjL(_E?e^RAwzH z;;F))wa3RA5hCg_i$$Ho*#RsbvlLaE^O7v*`ySJ@hf&~L?JH)6q$q}_S#z+`MQaS0 zy*Kf^e1Tf1U5W&=%?!ih%(E|?puxd4ygQVQ)-2AQz}WnLj9Tc=@+IUF^W%8|NeC0J zpd9*l-OzSJ{k^?!#<ew?O(tOlbxks*QBnjW;+7F=P7ArSt40{6m5q?zard2+=l9*1 zC+tM|E|Y5LImqxfYqqJC_kgYokX?IiPUx%*_7w(GvTL+l#N~ud)ZZ0S%N}&!d?l9X z*?X#*bDe<msYMY}g4^=O#3nGivs}2eLodiSLQ;b9YojM*L%i`6ON8N}35uKv%tBL+ z_oSEtZY_gMP3DpNy?sicFud|kx%?Q(8`z<t9bYz-V~L(x`FL4dG}z$1A)(g3@nA(y z_!!wn9TvG(hqXRl!cjHfn6wi`Tnr$rI1%@6TUSnu(kc!d<{Q*_sSS%BADi!o_$Z*o z?EA~8WJ`nJV@JLqcP%u)XrnB5w~w0wb8G(4cf8$jh2_FE(&;}h4}+Jvx&!**oJXH4 zm+hAZw?<<&*Jx4QZZf;nY*6a{?gye2dD{9@?8>2_*o-T4f$RbxzHwsqcOFax9vOLj z%I?q&|AW0{%shG--=9i}WruDJuMvP4c00tZuOJZGq7IDkJD_WFMN(Q?zcmC*0m{~v z7w8SefRrAzBlmQHzgsa>zua}Pm$Tr@kR7hd2Y|2kcGvLa&rFV@5tr5pO+yrbI49I{ zB@w~k_DPlMFQo<hY(^+ltyw4T1ZhbVvM-)^-V8J=Np0M1hxH+w&DG}%NE%q&+S}Ws zGW;mF0|2_$+)j-3ZcBp_TT}*DK&coi;6rxXYn2Y*%(3(G51gHyCxnlyvMYviD5lo7 zPP3{Y5IWmMT8V^yZZ+!@2)+Oq%9C$TTc9{b_mGDs<;v-E2Ml6%Jfr78Ow_NT2M}y2 z?J#!?H?#NSM5Up<sVQK-MdBa|9J_t26{suvdV4_w!z@1G)d+<2@W{{r)LV3gHOYLa z+aK#U8bH~1y51XEA;OyKw!LIvezm%TibQq(TtkuU{jsJ<*Pi8Vba8P9k>>p`kgPR$ z^^{Wn!R4<Hu3jLk&Ob+br}5h-d6!Ibv3gUrN}*30gDIC-SUDRut|0*F2N`l<V|ml^ zA-FN{91ZuFk33`3(*p`R)>Mhhiusk5KAUsh00V)aEUjES8ymkbZ66a75}??3<$N7M z1V|CHv(w^Km>3yTt<Nr6ySTX6+w&XbW9K89CO2}ScCwP2Qqu#xdB4fhzX@(#_cv_q z9Z$+He}-2KZC<7E%g)ZpA-A*j^71OQ?POnemK+s(P3zbAMRoRei?}=anQV8s00@;7 zr|Wus&L-Ntnw}UL2&t-rHe%Jv)d5_8f9&Lw#O2O&LDQ)WNAqETOW^@WiAaVQ>4vWI z?7rH%F9=Y|wMYj7K$zYbvM@0Po;3!5I*4H>7XeBB&OFEt_qG6Q;P<{CMoUXe{zc>z z6!lA4?G|V3ycdiVKvzU{4s|Bi+G2O(Iw;KiF-av0mLcrAiVdZvpVS+`;aI#TQp_rz z`}qVKlnmJ=&A6o5caDyl<Y_aOKQ13)_M!{hy5Jg~7$%p|nJ4pQwp?`f2{0(isd&;P zkw4#$heL`$@Pu4f_jPhw7iO5<^aLSA(w3w0|2bUZV5*+`G#6X$GOxLZYv&B(0L>@* zg(qhMu*=&N<m7k6N-8Uvd`mpm&1T*)_5T`KhQAyWZ=GKM^>}rvCC;>51or&m0J@WC zXJ<$5f+c5a+?>Zk*S~D+&qtzoP$xTxv8zh7_Vx6D(g%2&x<G~p=Rr{up7O`=g5qLP z&d)*tA0r=TYuXQ208#k%))weA@I{fWhQfTYd^>FNok7y`M=@2BZ|onykx)+u;OkCd zokIm04H$elSW!(WH^2R;ee?N?dZs?c_W-V_zN+rW5VZVZ<n((vNGPgi2>H0B)U3(r zDTQ)WOo#_zdwR--?Dj&H($wGBzt#ey(eVs>6$ccA)Q2T@y{mMy4Kcnl@ke0q9QYy` zr&FDby}ZhC%M`_Rpi7UzQ5Z3fI$yM^{hphQUg*izTXZeg=L;@21VpZp5o#*N9Og0M zHq2+kqo^If-cg%c$&k1gsM0^8<1P98DP_)G(8wNo>G5V6f~@b-pRY)&ShW)fgzyEw zyp;Wbe6c%WqCr7H{>9Y3NX}B$v-bjr<jaR-B_%I1*f7Ki3KmX;9}W4Wn;m4};F)wI ztNU~ys^bG?wERUgK++7^xpgQGTNTy$N>4WpLh#>W=Cq>r>g|UVk6V@{0C@~FjJ<W& zY-6Oe=vu_@)Y>oD`2rSHSvlpS*uwJmSMiPp357*=XofM)2j_=Moy<GRMe8G^#X?IX z(mCZH2*(D5SLGx(jf`wM71CQ5^>3hU7P9O_<5HqF+QXix-_rIh3sU)_Yh;w<D1+SU z6e#>5<+hZ%YZVrBq9JERMPRR`hNqdt!BNebS6M0Zln`;9QC3B_Oq+p$ftBU)UC_5{ z2p8qLQ16hf`F^|vJ8#ecK&vkx&S8QT?xm!1ph*J~habV8fJV=Yh3P8JLikpy`ONiO zo*JOO^$MmFS;gCDo$Fft0hoC?;szu4kU^y(%NeiKa<Y2LwPw}QBK6GkpYO4rn^VT^ zeRKkrDmZx-FMe#e0Jb@=MLSzlAubOl=+QPa?<5Z%0p&q5wW%yD;>K&gZ@IgwQ=XNw zg+E|8%U80_*BUX12DbsehfU0WSXvZk@ojN&QKu}JA+S!zZG;(^YgJ}bY|sPKD8E+* zji%)hCJ#C)B5(eBEbuet^0AERs>BZj!Lt^x=MqvB1U>0s!;Pw?n1m+NUWjCL?2ybd zVSjL5r7RuvWkI;~;NU=~II@Q)!FdIonT@x_XiY@6dZzO4+_NUd9Ex?!jlFI{No@hS zA7ESdYWe&gLu;re-!~H#(|TsT&<=*EV8<EJbm`sa4Yp=Q@Tai*ZUYRDk^1H@2k9;{ z{c91h7GSgh^mm_VdiR0kL9)NW8BGbYGJd*&3vph_^#!`wAZzGWKGZHzcrmc<7Mn7# zXACTo(~Q7QgS(Qc*@${@K5h1J7&!yYW8qFt@%K5Evjg!y!K^)kqM;#%XFIIhXN`1w zhr+L;zFd+V7k}<-9d_&9uD5Iji1fL+<}Gq4-Wpd%8H@&)CwiFuHmVn0{u9Eo{}r72 z<QXwo4NvoF*%+0wBinJ;>qk@V@1%R(CCNE_bdmdYJmt#84}%NK%W$H9JbVszCi!C< z<K@l3){b%gTx_QW*lQSug9)qy>rf6@><l#?x&f`w%aUgf)&RnYollcE@vJq{b<p!- z?wqq^{gh~eta$j69c$804#gT)6`bxKmP4_1Ch%@&G?=Q4&X~M&Q?hbq-rbJLh<v4I zWQf*rkCOXWCJbjpC0)f0%e?6*s?PAUDr3$i9>97r9+^92ytUloEsHezEO)KBRpyIf zE(Kqw9a@cLFm}MAD320%abd+^p;=BSHun;T{@(kJg{0q|CzLvPc{m!vj}N5VFaD0; zo*izJ^q67fvlA*#mNc$@O(sYA#I{v|^!=MTPhqH2aqLk6%lg)qt0cD|BEASXbi>_5 zn$X?G0c*X@z#SRhHr?qv{QbAQ3+IPGR!Ex3DC`cU9G{+%6pfqfFS5=UG`Q#>(bl)T z1i`J{ery_jt`Vl{BihFE+k<|ZXQQB)I&)dD+JgKg<jy&O?gW0XTj~6=NhvDS#Ry}| zv%g-XH-8r_nK*VFUFW($?lP5VjuE{sf6?|D&HL&5)y8$z&5zN=y}EBY<MPxqftfL> zwp>(r$t1jq=SI5Br&EZLc+kF_q@d+g%r7c}Ot{lBDc~#Dq3Vv<hn7|z*Y0P&J|=XJ zI{ET1)>hVf0p4)W1wXy_{RVID^3J8bcJJP>%&AcVxvzyB)!Di>E?$v}IbRHN7UDMT zm+%d+`Kvw1ew~WKUjkvXEAqh40mKXF0T~~_M@IhwXRT;ndvPw%fbe6-H26OGA!UbW z&#vz^xt9CLvcv;FR?{ThbgF=r1LN}IBCpj^J(!v=k2|kA(BFRzBJNuzkx?wgksOVw z*=Z(;LcH%H+NR~IEq|BO)H`H!>NI#=46VKOb~BGmc~7R=<FrP0l0nbH<mHlMh-ptO zRU)07Lya=y3b^`g&Gq3!&k}L6Ts}1{r|necjF>N`BE4SL)OfYkGmnD%HJ6q)xxSP| zc55fO+v=>(voUd5^v=(gII&s^Yiny$<nw`n0T^Sse}#vmoIBe@w)mDiV|}uSnQ|e3 zbI!aE9MTQtfD?iD?vbah-I9CzCXdX<))s;<gG$fM4GGX(7vZP|+t|MT1f1ZIotmy# z_T-f!GI8q%=U;o6i2Py%3pk<=Cd$7&5?F4l1JZ!nEuZugDjQ2~i5}>rr3<}mW6<@4 z1M%#$?~5O*FY->EhM4uclH|M3imZk0F0{iK2~h`=FBZMwxg&aLJEnbx65*sJO5ACI z=*minu(7kw#b~samX?#bghJORFyVDx5J_}*cmGhc#^8+f^vFU&H!ReMUu&2C1_iV7 zSM>)w_IK&cbAYpe_3Qa!QTG)SYu2$xx!AV7R?Nd|uT4Q%T<%5nGmg<-@PxO9d%(V~ zbc6I3#FRZfJzHB_@Mm|H77(lz=_YKLrM<oa`qAQ(tr={b2rf3#vpVojjMR-4UU4D^ z2D+A03_cqd8GAqhORDKOcf>nCUIgDyI2hrFHpBcrn`FY8CiDMa?Opjll=~MSb0ah@ zmuYNesbR=8gwSLuQ<yBF5@WfP63G^$l%;HCOJv`!z0DS4xD!JpYniN>a7CsPg~s0Z zjC=oq@9+2dg_qa!n&+91&*z-;exGyRXD}zNAtWG;RN!}m{^7$1`{j|q`k<b#ap&tt z3>3NJ^>WT;UX1*xkF+Dr`I(+Pnle7?@F+i|>2FQYIESbg0FCkHbbY0+qZNb3znQ|n z7|O20^u|8-K5==f7@6x{eFIYm*vZikHlZ@+pbsm3y)3S!ns56HfGMME08*AZ%HF$H zHBR`=`iAW#e_A8DTo@ulcL!{@n!s?MY7iK5t%}DgV4_a-yzB9Fh{v)OJ|*yHgn4Gu zR_C56;TXj$Mkx~o(1hB4p*hU@$JMIU+tyih<^*o1MsNXiU6RIDnv*QZQb+^b{LtXw zq3Z7`2$2GqOH*7NW`pdGNQL$+zVk+^C%(8BX!GVgQHvdCybFEjP<?UHUUK;veC`<A z#n*RXK-Kl(jR3$l(VXmc6~L}wcUq5%eLfVEd#UWbzx-Vpa=czn+4JX^*x3OBJE!Rj zXJw2(!n?U~#*a!pR&TY>91T(SHrYYjR;(;IaX&u|I(OFXO)EV-HYP7)0j6^fjg08J z;6Zc5n7DYc;?S$O+e;tkmS7#-J4KI!H+=i;n|mu?6gR+x!C>b(7EgLXyQ<bW9(w|f zbk2dWa4Y2;Rgv?J#T4Y+H7%+rFTbx-0M*-Kn`fGqAc!%OM}0WeRde%N<J(~Tv_fW9 zE;&A5`vOF2Q`Q0BPWfNH3~>SgLuf~_{Cu(+BRM@8xO19%IJ6U7hIR7zX3hI?n}i9$ z445uz6-b3b@g*T?@Y~U(%+CwUX7#fUW2NTce97^ZBaZWmzCE7-c?#SrfKY^?t#7VI z7ujnvWO(w2UXHTtWaWz&I-*7r{=?~&{)U$gJwz?fo^4Xe!W_!G>?@w0>TydmmL!cm zrcn4aab2ZYNn<58Ry!Jmo{^ubJw{$orztww*=lxe^GzupegFg>$j}+ZyKM}-lA6-t zh$PpE-OGm#F^?8ncLuq!;KuSP3wf8rr(aXYBGlSn2f5uqTBH3b`YT~6&A@8v&_Ylo z<Uw9*koW+$fGyN5o6!9UW}A{|t5BB@NGc3{8ic$vO>wc~8;!q`&r_*$0&B=*i_-r0 z0|UxK-&MrEKG(TVDORU^^Z437ajAs_d+5d<;11wCvlA?#KB}AnYQY+57zp-Y^y>q% zEr@MJME(}>F7oZiuhPcds<P(zkECe*b>JZ4u`Y~iwdD&`v92k;Ys<T5%C{UdSM&E0 zt?QdN$($@@M0XcexVhe{OMiR!xqADUiGKyg7h9xzRgO%$htqBilj_0a>M!%#_9?w0 z@5O2+Jqu0#qlgLGTP{3PbB}URQ3q3e_>=XCiZE_1Vfpc7jiTWAQ)UUh^6$9Yp$!uW z*w=uU#N({qt<Krjz*nH*Y{eee9sC*P{Z-Yxb*F@#SI>+=eIJLzF%SXl)|Y>V>gQ*$ zVoXB2x&@TQnH;Q*iu)AcmTgEJ?~1E@)8_!Lt5zvE8aQB3tWRhYXz|*${&AA0<_L@~ zU48P^UR<S`@Jr-6r8eN|)e2DF`P~>3R`P3YEx?2!n;=I4lLRIhuoTln^<@Vjn^u36 z3?W1!*vJKJ&r5{f%Up;-yNkzkK(C%;|Bk^p+NvL!E1%shWrLDFT?o$wwk&d7^J#q2 zYghGjw&#=b`}t6iy)TiN*`?a8C7kqE;M(#)WwDPR%e|V(9tQ5m4ltOhz$pk!Hs(b& z5_08hy~g#gnUmvD7@=JfF(AeZP7Q7b1_$eA2XJlLSU`B}>-4l8&HIA92TD4M#XnZ! z0G@2nZJS*TX~=K!y39#OM20S-xH(<+(NYGW8R}_i6l-a}qou*Mfp44@7z6L@jLRhP z6C%teFZucc2>}n`@-83Gi|XIRk)MI3d69_5Zo*t^Quh0vtqQg-y|d6l5(SyKIc**j zir0x0*V%Lsn#;Q6mw4hr(*Er_!!fMp=(=sZsG`;0%Cv@w9fV8ZlD&4778e6ZRki{f z26LD(f3Sv!-`VlSnYRgLoq(PZw4mct19^n+0xuYNrZZL&hAwaPQ^ytB#Qc;C<%On? z=rcpWu!F8#z?p=hp&>zM^TioJs5xd<*46^QN=iyh75a270kBFK<NlPas;w1II^^QZ zP~LIa=PZ<~GF?2E&{551*A+VM)x=#1cRA#(6kYw$@`<1j64(1>q0!Gz{>spo1$Ywp zLC{iN<m5ALWywPBSzE#y^V2`h6E3vW-M`#l;#<d>x4dPvS8L|dW`ybaGP5hC{UAaG zlht}%!&c&E-@pM^LwnvDn*Q0=7RfgKw7150-}hkf-!R;Gz)PwY5u4e0&FtaP-#?#< z(JpF=M#B1(mUjO@{jm=^g_4Q__~5Z!xgga1{5fc?@!cqdvsy*bHuumgw}PIB99yo$ z>=Y7K79e}<IPivDUg|(yDF^%9vintb8YXs$-hT6Cp(XInr02@dX@-Yl45}lb6tYDu zFqFz}iuC@NabQ`B66`wnbYRiXX;oDRseZ+lif*1Q$fF=kx`Tjlh<B~>zjjU3RV3Md zhxdznv9S*vcw5yN#k~A^X5(C%55}Dun`Ok_2PBFUq!+5<?&-``zn`A1l!|P<eBIOq zVkjm%v<(Z4B-}bJ3C9B$*$zg^v#Th;;t@>8G}bLj{qYH3yYBc1uTDC!He8TBm2lAs ziL$f!v2@eFDel9aaH4L7$3A1hl=g|1ZFk78`cf_kBdi{nRrv-5)vw%Uu~<+^A+33& zPr!Zu5kvAIiv<(~#Bks=zAWVB<@q$Q@7~^&khMe_Y|2oX!kgy39T*<gA`%7Zr{T8u z(}m8E<w>K5I><|=NF>`PX8nR$nQEL^m;Rt)OK`FubO@>QLy~fB#EHJ1{ir}sQwka_ zO?SMLI|cHAbjJY}OB+5dIhkRtc=7&+?WkTYLj$iX?(X4Y8no;%<;NSDDd37&?)}1V zZf>f^fNI^nLW_b0#Ac(Kj+&~tO{$9>^%WJul8emxMg=&l6g%Sn8L*Cbmmh=iND-_9 zw5CWUW>V`iZI0EBIav8Yi<pLX^2xNEff>##%-1yYI@UIx(OH_!VyJyXz(atkte|aj zPTBGrKAuO6iAXt^8h}nl9u1W+I=Kkh5Px$tBo1l$bC9rzE~ozlP&&U5QN&?t%@y$$ zqOnf5<O&dl<cN@~KRfnx{!U(bL)yHS+p`rm`l2yJqr$Ss^epqGa)ECe!SEfv)AXr? zQXHN#m_N9Srx4Wm&Kp=3XrOTW3&rKQJx}cDZva8{y}WdXZ;VL%DGuqTjPTJuB)A*4 zjMmy3IeUqTO_}YG^555l9~RFzo~kCPuI+I$ZFgc;ZqAK0e9u<!l+4$3ALdnsdb7sS zaM~3Qk0M`sPdCe9)VgnP*+YqNK$;vx(|j$ugre_tzzNKo(=RA#mBr}KW(WRQSV<V| z`*9+B2^g`)yv&@Z_&#f=zY2YO!OTCrUNaE;SgrIxQF@lqV~a-}cJW|=B4(m;YVFxU z%)bc1gC?|J87uaCItkMY{s1AIDc0-U?Qv{^z=M#Ef&-fOcE@nqJ?<kC9P~JJG5*%b z9myu~*x2!ksfWPi)v{^BlM-#pi+6#mCIy)Y)10)CMtgSe$lDk_i(7ys#l?^iNA2l^ z;!_%(kBCvLWq`ZbK7VVvS5Y+FhYfMnAr~;qp13m{@#Rfi3}V=eZwEqL>A`Ed*Lcg+ z*_Kgt;1wGg9pB$>Q?#$6c1j3ld>CzRqZ2|vHt7@+5p1M1Uzu^7!(Kb$dHK35StTVA z5{J-sZ+bIn#35%O?nIB9x=44bHb@0;*5cLz1hPh0tgUCFL5uO?%vwa0(DApI)O*nd z7VP}*MdyuzyUIIdpyH~3N142fY%gdmjnB@$A?;?YufKp|_0b+C(_6evh)}AgS4N|W zWzBUezx-iz_%W;pajDPx-4UZGVPMG1%X21pq)9y|9-E`#;7p#DfZB^u?564@&aHZ| zs+-p>pX?O+Bix(jFo?>^<=UMWiH9p{+$si9hivHf-3GrQRGd7~aFB^io&!LU*g}@p zXx}@VcDmX9JW8?ur1Mh2?lX?2i}I0r-o1k{%^yZawtuf+)aoM|RG<fMEb1`*jG>}v zdy8CvxVH+V*V(|v0=f;XFIAY7uBnRd;XevO=s4ja7Rw}iX(!G}U~8d-x_HcN`;LjW zOlw>9$Xpob_?`PT{ya_;1L9ii3(o<kZ!IdBO`rx%cmm|$ORuNZLGWhni_#*<-$?YI zE)5N;y6p-k8oMIZ3%mvaGgJ(CfRMrU7)T^Y;0;Cf8}d2Z$Lhf%*BpnhRVX}6A+pW- z<^^;UsXX$gt<S`aHDJziND!>3Bc576qMKLaMx==#$~yQAyu#jxoYu+K<-Uhu#o#bF z-HnW1frSg4CV9bzXjow*upH)3&XsGTWkmfa&t}9;98h2>19T+KLhp6v@vv0+$Q~LI z5nOOE<&*aEs^z{!J#{KXq>vx9<y3IH%}!Useo40BX1>mNUe@^s9>Cke<<v+@gCD^Y zKvaT7CziUx<8-0a%xwFgBQp`S!CZN9JMd{?Z%e&)MKp?n9Wvql;zP{gYl+804BL~t zn|%)Y>a6gDrgJXo9Dll5h|HBaEE(b{6}<Y|O{@6kq$TXGx`7`Vp?OjJ$H<zs143D$ zgcL%GwbofXWs#}@mF&>e6>r_B3PqMC?kcxpCrN+G%<xyb4jq`r*>p4E!}D9B36L>t z^61I^W{%$Kep3tsGt%=;PPk~C0Gc+w893@jCFc4wkbeuj>f=+1jWW@dx~O%f%fDP! zOW+mZc_?md>YULvp;+#(Ia><1OpcywzscBT>z1mN!sh;JEMhbH?_Z5Y4*&Buxp*0P z9i#mIFA&2kztLRPST)0Am^>AZ9UfE>pzu*!x9yf6i-4d>t}yh)_NdNh1G1V@NzMnH z`*Sf@sWb<|(z&h@Uc%&8%IG=H^kCw8vhYU23O-g^Rq_qD-V!dSuzxaR2t=w3fO+n9 z<RbsSLe6nwz-(<?PT~LE1`F}uf^6)D|8~efr{e!VGOJSc6!%iqmOUFo%E`w~kCo^- G-2NYgE#glA literal 0 HcmV?d00001 diff --git a/dev/reference/forecast_types.html b/dev/reference/forecast_types.html index 5797e7be9..c3edff4d0 100644 --- a/dev/reference/forecast_types.html +++ b/dev/reference/forecast_types.html @@ -44,45 +44,6 @@ <h1>Documentation template for forecast types</h1> </div> - <div class="section level2"> - <h2 id="forecast-types-and-input-formats">Forecast types and input formats<a class="anchor" aria-label="anchor" href="#forecast-types-and-input-formats"></a></h2> - <p>Various different forecast types / forecast formats are supported. At the -moment, those are:</p><ul><li><p>point forecasts</p></li> -<li><p>binary forecasts ("soft binary classification")</p></li> -<li><p>nominal forecasts ("soft classification with multiple unordered classes")</p></li> -<li><p>Probabilistic forecasts in a quantile-based format (a forecast is -represented as a set of predictive quantiles)</p></li> -<li><p>Probabilistic forecasts in a sample-based format (a forecast is represented -as a set of predictive samples)</p></li> -</ul><p>Forecast types are determined based on the columns present in the input data. -Here is an overview of the required format for each forecast type: - - </p><div style="text-align: left"> - <img src="figures/required-inputs.png" style="width:750px;max-width:100%;"></div> -<p><em>All forecast types</em> require a data.frame or similar with columns <code>observed</code> -<code>predicted</code>, and <code>model</code>.</p> -<p><em>Point forecasts</em> require a column <code>observed</code> of type numeric and a column -<code>predicted</code> of type numeric.</p> -<p><em>Binary forecasts</em> require a column <code>observed</code> of type factor with exactly -two levels and a column <code>predicted</code> of type numeric with probabilities, -corresponding to the probability that <code>observed</code> is equal to the second -factor level. See details <a href="scoring-functions-binary.html">here</a> for more information.</p> -<p><em>Nominal forecasts</em> require a column <code>observed</code> of type factor with N levels, -(where N is the number of possible outcomes), a column <code>predicted</code> of type -numeric with probabilities (which sum to one across all possible outcomes), -and a column <code>predicted_label</code> of type factor with N levels, denoting the -outcome for which a probability is given. Forecasts must be complete, i.e. -there must be a probability assigned to every possible outcome.</p> -<p><em>Quantile-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>quantile_level</code> of type -numeric with quantile-levels (between 0 and 1).</p> -<p><em>Sample-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>sample_id</code> of type -numeric with sample indices.</p> -<p>For more information see the vignettes and the example data -(<a href="example_quantile.html">example_quantile</a>, <a href="example_sample_continuous.html">example_sample_continuous</a>, <a href="example_sample_discrete.html">example_sample_discrete</a>, -<code><a href="example_point.html">example_point()</a></code>, <a href="example_binary.html">example_binary</a>, and <a href="example_nominal.html">example_nominal</a>).</p> - </div> <div class="section level2"> <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows @@ -103,13 +64,12 @@ <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href=" is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> then thinks that this column was relevant in defining the forecast unit.</p> <p>In order to avoid issues, we recommend setting the forecast unit explicitly, -usually through the <code>forecast_unit</code> argument in the <code><a href="as_forecast.html">as_forecast()</a></code> -functions. This will drop unneeded columns, while making sure that all -necessary, 'protected columns' like "predicted" or "observed" are retained.</p> +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> </div> - </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> - </nav></aside></div> + </main></div> <footer><div class="pkgdown-footer-left"> diff --git a/dev/reference/get_coverage.html b/dev/reference/get_coverage.html index 846f035ec..8c56b5dd5 100644 --- a/dev/reference/get_coverage.html +++ b/dev/reference/get_coverage.html @@ -1,6 +1,6 @@ <!DOCTYPE html> <!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Get quantile and interval coverage values for quantile-based forecasts — get_coverage • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Get quantile and interval coverage values for quantile-based forecasts — get_coverage"><meta name="description" content="For a validated forecast object in a quantile-based format -(see as_forecast() for more information), this function computes: +(see as_forecast_quantile() for more information), this function computes: interval coverage of central prediction intervals quantile coverage for predictive quantiles the deviation between desired and actual coverage (both for interval and @@ -28,7 +28,7 @@ (can be either interval or quantile coverage) and the actual coverage. For example, if the desired coverage is 90% and the actual coverage is 80%, the coverage deviation is -0.1."><meta property="og:description" content="For a validated forecast object in a quantile-based format -(see as_forecast() for more information), this function computes: +(see as_forecast_quantile() for more information), this function computes: interval coverage of central prediction intervals quantile coverage for predictive quantiles the deviation between desired and actual coverage (both for interval and @@ -97,7 +97,7 @@ <h1>Get quantile and interval coverage values for quantile-based forecasts</h1> <div class="ref-description section level2"> <p>For a validated forecast object in a quantile-based format -(see <code><a href="as_forecast.html">as_forecast()</a></code> for more information), this function computes:</p><ul><li><p>interval coverage of central prediction intervals</p></li> +(see <code><a href="as_forecast_quantile.html">as_forecast_quantile()</a></code> for more information), this function computes:</p><ul><li><p>interval coverage of central prediction intervals</p></li> <li><p>quantile coverage for predictive quantiles</p></li> <li><p>the deviation between desired and actual coverage (both for interval and quantile coverage)</p></li> @@ -135,7 +135,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-by">by<a class="anchor" aria-label="anchor" href="#arg-by"></a></dt> diff --git a/dev/reference/get_duplicate_forecasts.html b/dev/reference/get_duplicate_forecasts.html index f827d3ec7..ad2ac723e 100644 --- a/dev/reference/get_duplicate_forecasts.html +++ b/dev/reference/get_duplicate_forecasts.html @@ -60,8 +60,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> diff --git a/dev/reference/get_forecast_counts.html b/dev/reference/get_forecast_counts.html index f9ef64b29..cb54b21cc 100644 --- a/dev/reference/get_forecast_counts.html +++ b/dev/reference/get_forecast_counts.html @@ -73,7 +73,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-by">by<a class="anchor" aria-label="anchor" href="#arg-by"></a></dt> diff --git a/dev/reference/get_forecast_type.html b/dev/reference/get_forecast_type.html index b0cf64116..988f72bbe 100644 --- a/dev/reference/get_forecast_type.html +++ b/dev/reference/get_forecast_type.html @@ -54,7 +54,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> </dl></div> <div class="section level2"> diff --git a/dev/reference/get_forecast_unit.html b/dev/reference/get_forecast_unit.html index c4d814db5..127e6b84e 100644 --- a/dev/reference/get_forecast_unit.html +++ b/dev/reference/get_forecast_unit.html @@ -69,8 +69,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> </dl></div> <div class="section level2"> @@ -98,9 +98,9 @@ <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href=" is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> then thinks that this column was relevant in defining the forecast unit.</p> <p>In order to avoid issues, we recommend setting the forecast unit explicitly, -usually through the <code>forecast_unit</code> argument in the <code><a href="as_forecast.html">as_forecast()</a></code> -functions. This will drop unneeded columns, while making sure that all -necessary, 'protected columns' like "predicted" or "observed" are retained.</p> +using the <code>forecast_unit</code> argument. This will simply drop unneeded columns, +while making sure that all necessary, 'protected columns' like "predicted" +or "observed" are retained.</p> </div> </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2> diff --git a/dev/reference/get_metrics.forecast_binary.html b/dev/reference/get_metrics.forecast_binary.html index 9a0a7e1ff..629c4fe65 100644 --- a/dev/reference/get_metrics.forecast_binary.html +++ b/dev/reference/get_metrics.forecast_binary.html @@ -64,7 +64,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values, see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>).</p></dd> <dt id="arg-select">select<a class="anchor" aria-label="anchor" href="#arg-select"></a></dt> @@ -112,7 +112,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> brierscore <- (observed - predicted)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(brierscore)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba2740></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc441010></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $log_score</span> @@ -123,7 +123,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> logs <- -log(1 - abs(observed - predicted))</span> <span class="r-out co"><span class="r-pr">#></span> return(logs)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba1908></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc4401d8></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-in"><span><span class="fu"><a href="get_metrics.html">get_metrics</a></span><span class="op">(</span><span class="va">example_binary</span>, select <span class="op">=</span> <span class="st">"brier_score"</span><span class="op">)</span></span></span> @@ -135,7 +135,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> brierscore <- (observed - predicted)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(brierscore)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba2740></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc441010></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-in"><span><span class="fu"><a href="get_metrics.html">get_metrics</a></span><span class="op">(</span><span class="va">example_binary</span>, exclude <span class="op">=</span> <span class="st">"log_score"</span><span class="op">)</span></span></span> @@ -147,7 +147,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> brierscore <- (observed - predicted)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(brierscore)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba2740></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc441010></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> </code></pre></div> diff --git a/dev/reference/get_metrics.forecast_nominal.html b/dev/reference/get_metrics.forecast_nominal.html index 5b9ce0e65..a538689a4 100644 --- a/dev/reference/get_metrics.forecast_nominal.html +++ b/dev/reference/get_metrics.forecast_nominal.html @@ -61,7 +61,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values, see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>).</p></dd> <dt id="arg-select">select<a class="anchor" aria-label="anchor" href="#arg-select"></a></dt> @@ -105,7 +105,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> logs <- -log(pred_for_observed)</span> <span class="r-out co"><span class="r-pr">#></span> return(logs)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc52980ec8></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626c9ce0070></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> </code></pre></div> diff --git a/dev/reference/get_metrics.forecast_point.html b/dev/reference/get_metrics.forecast_point.html index 067e64ebb..5c78360b7 100644 --- a/dev/reference/get_metrics.forecast_point.html +++ b/dev/reference/get_metrics.forecast_point.html @@ -97,7 +97,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values, see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>).</p></dd> <dt id="arg-select">select<a class="anchor" aria-label="anchor" href="#arg-select"></a></dt> @@ -143,7 +143,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> {</span> <span class="r-out co"><span class="r-pr">#></span> return(ae(actual, predicted)/abs(actual))</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc550ceb38></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cdf8d760></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:Metrics></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-in"><span></span></span> diff --git a/dev/reference/get_metrics.forecast_quantile.html b/dev/reference/get_metrics.forecast_quantile.html index f8d924f80..d63b3e646 100644 --- a/dev/reference/get_metrics.forecast_quantile.html +++ b/dev/reference/get_metrics.forecast_quantile.html @@ -109,7 +109,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values, see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>).</p></dd> <dt id="arg-select">select<a class="anchor" aria-label="anchor" href="#arg-select"></a></dt> @@ -191,7 +191,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> return(reformatted$wis)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc56d4c188></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cbe3e440></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> </code></pre></div> diff --git a/dev/reference/get_metrics.forecast_sample.html b/dev/reference/get_metrics.forecast_sample.html index 6fcc69362..b7fc24242 100644 --- a/dev/reference/get_metrics.forecast_sample.html +++ b/dev/reference/get_metrics.forecast_sample.html @@ -88,7 +88,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values, see <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>).</p></dd> <dt id="arg-select">select<a class="anchor" aria-label="anchor" href="#arg-select"></a></dt> @@ -141,7 +141,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> return(res)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57067b00></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626c5a06138></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $dss</span> @@ -150,7 +150,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> assert_input_sample(observed, predicted)</span> <span class="r-out co"><span class="r-pr">#></span> scoringRules::dss_sample(y = observed, dat = predicted, ...)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc55ab8df0></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626c9da6bb8></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $crps</span> @@ -182,7 +182,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> return(crps)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54553238></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd6a3b30></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $overprediction</span> @@ -192,7 +192,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> ...)</span> <span class="r-out co"><span class="r-pr">#></span> return(crps$overprediction)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57e45db8></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd07aed8></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $underprediction</span> @@ -202,7 +202,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> ...)</span> <span class="r-out co"><span class="r-pr">#></span> return(crps$underprediction)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57e490c0></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd07a3b0></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $dispersion</span> @@ -212,7 +212,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> ...)</span> <span class="r-out co"><span class="r-pr">#></span> return(crps$dispersion)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57e48528></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd07d6b8></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $log_score</span> @@ -222,7 +222,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> scoringRules::logs_sample(y = observed, dat = predicted, </span> <span class="r-out co"><span class="r-pr">#></span> ...)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57e47a00></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd07cb90></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $ae_median</span> @@ -234,7 +234,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> ae_median <- abs(observed - median_predictions)</span> <span class="r-out co"><span class="r-pr">#></span> return(ae_median)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc55b6ea58></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd47e540></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-out co"><span class="r-pr">#></span> $se_mean</span> @@ -245,7 +245,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> se_mean <- (observed - mean_predictions)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(se_mean)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc57e49f08></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cd07f098></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> </code></pre></div> diff --git a/dev/reference/get_metrics.html b/dev/reference/get_metrics.html index bdb1a30a7..d9fccc2a4 100644 --- a/dev/reference/get_metrics.html +++ b/dev/reference/get_metrics.html @@ -1,7 +1,7 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Get metrics — get_metrics • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Get metrics — get_metrics"><meta name="description" content="Generic function to to obtain default metrics availble for scoring or metrics +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Get metrics — get_metrics • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Get metrics — get_metrics"><meta name="description" content="Generic function to to obtain default metrics available for scoring or metrics that were used for scoring. -If called on forecast object it returns a list of functions that can be +If called on a forecast object it returns a list of functions that can be used for scoring. If called on a scores object (see score()), it returns a character vector with the names of the metrics that were used for scoring. @@ -9,9 +9,9 @@ See the documentation for the actual methods in the See Also section below for more details. Alternatively call ?get_metrics.&lt;forecast_type&gt; or -?get_metrics.scores."><meta property="og:description" content="Generic function to to obtain default metrics availble for scoring or metrics +?get_metrics.scores."><meta property="og:description" content="Generic function to to obtain default metrics available for scoring or metrics that were used for scoring. -If called on forecast object it returns a list of functions that can be +If called on a forecast object it returns a list of functions that can be used for scoring. If called on a scores object (see score()), it returns a character vector with the names of the metrics that were used for scoring. @@ -60,8 +60,8 @@ <h1>Get metrics</h1> </div> <div class="ref-description section level2"> - <p>Generic function to to obtain default metrics availble for scoring or metrics -that were used for scoring.</p><ul><li><p>If called on <code>forecast</code> object it returns a list of functions that can be + <p>Generic function to to obtain default metrics available for scoring or metrics +that were used for scoring.</p><ul><li><p>If called on a <code>forecast</code> object it returns a list of functions that can be used for scoring.</p></li> <li><p>If called on a <code>scores</code> object (see <code><a href="score.html">score()</a></code>), it returns a character vector with the names of the metrics that were used for scoring.</p></li> @@ -87,11 +87,6 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dd><p>Additional arguments passed to the method.</p></dd> </dl></div> - <div class="section level2"> - <h2 id="details">Details<a class="anchor" aria-label="anchor" href="#details"></a></h2> - <p>See <code><a href="as_forecast.html">as_forecast()</a></code> for more information on <code>forecast</code> objects and <code><a href="score.html">score()</a></code> -for more information on <code>scores</code> objects.</p> - </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> <div class="dont-index"><p>Other get_metrics functions: diff --git a/dev/reference/get_pit_histogram.html b/dev/reference/get_pit_histogram.html index db5777995..dfdae6fca 100644 --- a/dev/reference/get_pit_histogram.html +++ b/dev/reference/get_pit_histogram.html @@ -77,7 +77,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-num-bins">num_bins<a class="anchor" aria-label="anchor" href="#arg-num-bins"></a></dt> diff --git a/dev/reference/get_protected_columns.html b/dev/reference/get_protected_columns.html index 770868680..f9bbbc81b 100644 --- a/dev/reference/get_protected_columns.html +++ b/dev/reference/get_protected_columns.html @@ -57,8 +57,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> </dl></div> <div class="section level2"> diff --git a/dev/reference/illustration-input-metric-nominal.html b/dev/reference/illustration-input-metric-nominal.html new file mode 100644 index 000000000..144ac36d9 --- /dev/null +++ b/dev/reference/illustration-input-metric-nominal.html @@ -0,0 +1,71 @@ +<!DOCTYPE html> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Illustration of required inputs for nominal forecasts — illustration-input-metric-nominal • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Illustration of required inputs for nominal forecasts — illustration-input-metric-nominal"><meta name="description" content="Illustration of required inputs for nominal forecasts"><meta property="og:description" content="Illustration of required inputs for nominal forecasts"><meta name="robots" content="noindex"></head><body> + <a href="#main" class="visually-hidden-focusable">Skip to contents</a> + + + <nav class="navbar navbar-expand-lg fixed-top bg-light" data-bs-theme="light" aria-label="Site navigation"><div class="container"> + + <a class="navbar-brand me-2" href="../index.html">scoringutils</a> + + <small class="nav-text text-danger me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="In-development version">1.2.2.9000</small> + + + <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation"> + <span class="navbar-toggler-icon"></span> + </button> + + <div id="navbar" class="collapse navbar-collapse ms-3"> + <ul class="navbar-nav me-auto"><li class="active nav-item"><a class="nav-link" href="../reference/index.html">Reference</a></li> +<li class="nav-item"><a class="external-link nav-link" href="https://raw.githubusercontent.com/epiforecasts/scoringutils/main/inst/manuscript/manuscript.pdf">scoringutils paper</a></li> +<li class="nav-item dropdown"> + <button class="nav-link dropdown-toggle" type="button" id="dropdown-articles" data-bs-toggle="dropdown" aria-expanded="false" aria-haspopup="true">Articles</button> + <ul class="dropdown-menu" aria-labelledby="dropdown-articles"><li><a class="dropdown-item" href="../articles/Deprecated-functions.html">Deprecated functions</a></li> + <li><a class="dropdown-item" href="../articles/Deprecated-visualisations.html">Deprecated Visualisations</a></li> + <li><a class="dropdown-item" href="../articles/scoring-rules.html">Scoring rules in `scoringutils`</a></li> + </ul></li> + </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search"> + <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li> +<li class="nav-item"><a class="external-link nav-link" href="https://github.com/epiforecasts/scoringutils/" aria-label="GitHub"><span class="fa fab fa-github fa-lg"></span></a></li> + </ul></div> + + + </div> +</nav><div class="container template-reference-topic"> +<div class="row"> + <main id="main" class="col-md-9"><div class="page-header"> + + <h1>Illustration of required inputs for nominal forecasts</h1> + <small class="dont-index">Source: <a href="https://github.com/epiforecasts/scoringutils/blob/main/R/documentation-templates.R" class="external-link"><code>R/documentation-templates.R</code></a></small> + <div class="d-none name"><code>illustration-input-metric-nominal.Rd</code></div> + </div> + + <div class="ref-description section level2"> + <p>Illustration of required inputs for nominal forecasts</p> + </div> + + + <div class="section level2"> + <h2 id="input-format">Input format<a class="anchor" aria-label="anchor" href="#input-format"></a></h2> + <p></p><div style="text-align: left"> + <img src="figures/metrics-nominal.png" style="width:750px;max-width:100%;"></div> + </div> + + </main></div> + + + <footer><div class="pkgdown-footer-left"> + <p>Developed by <a href="https://followtheargument.org/" class="external-link">Nikos Bosse</a>, <a href="https://www.samabbott.co.uk/" class="external-link">Sam Abbott</a>, Hugo Gruson, Sebastian Funk.</p> +</div> + +<div class="pkgdown-footer-right"> + <p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.1.1.9000.</p> +</div> + + </footer></div> + + + + + + </body></html> + diff --git a/dev/reference/index.html b/dev/reference/index.html index feec70797..d72a83c46 100644 --- a/dev/reference/index.html +++ b/dev/reference/index.html @@ -114,16 +114,16 @@ <h2 id="create-forecast-objects">Create forecast objects<a class="anchor" aria-l <dl><dt> - <code><a href="as_forecast.html">as_forecast</a></code> + <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code> </dt> - <dd>General information on creating a <code>forecast</code> object</dd> + <dd>Create a <code>forecast</code> object for binary forecasts</dd> </dl><dl><dt> - <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code> + <code><a href="as_forecast_doc_template.html">as_forecast_doc_template</a></code> </dt> - <dd>Create a <code>forecast</code> object for binary forecasts</dd> + <dd>General information on creating a <code>forecast</code> object</dd> </dl><dl><dt> <code><a href="as_forecast_generic.html">as_forecast_generic()</a></code> @@ -801,6 +801,12 @@ <h3 id="misc-internal-functions">Misc internal functions<a class="anchor" aria-l <dd>Illustration of required inputs for binary and point forecasts</dd> </dl><dl><dt> + <code><a href="illustration-input-metric-nominal.html">illustration-input-metric-nominal</a></code> + + </dt> + <dd>Illustration of required inputs for nominal forecasts</dd> + </dl><dl><dt> + <code><a href="illustration-input-metric-quantile.html">illustration-input-metric-quantile</a></code> </dt> diff --git a/dev/reference/is_forecast.html b/dev/reference/is_forecast.html index 12954f4e9..8b7b938c7 100644 --- a/dev/reference/is_forecast.html +++ b/dev/reference/is_forecast.html @@ -1,11 +1,9 @@ <!DOCTYPE html> -<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Test whether an object is a forecast object — is_forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Test whether an object is a forecast object — is_forecast_binary"><meta name="description" content="Test whether an object is a forecast object (see as_forecast() for more -information). -You can test for a specific forecast_* class using the appropriate -is_forecast_* function."><meta property="og:description" content="Test whether an object is a forecast object (see as_forecast() for more -information). -You can test for a specific forecast_* class using the appropriate -is_forecast_* function."><meta name="robots" content="noindex"></head><body> +<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Test whether an object is a forecast object — is_forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Test whether an object is a forecast object — is_forecast_binary"><meta name="description" content="Test whether an object is a forecast object. +You can test for a specific forecast_&lt;type&gt; class using the appropriate +is_forecast_&lt;type&gt; function."><meta property="og:description" content="Test whether an object is a forecast object. +You can test for a specific forecast_&lt;type&gt; class using the appropriate +is_forecast_&lt;type&gt; function."><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -46,10 +44,9 @@ <h1>Test whether an object is a forecast object</h1> </div> <div class="ref-description section level2"> - <p>Test whether an object is a forecast object (see <code><a href="as_forecast.html">as_forecast()</a></code> for more -information).</p> -<p>You can test for a specific <code>forecast_*</code> class using the appropriate -<code>is_forecast_*</code> function.</p> + <p>Test whether an object is a forecast object.</p> +<p>You can test for a specific <code>forecast_<type></code> class using the appropriate +<code>is_forecast_<type></code> function.</p> </div> <div class="section level2"> diff --git a/dev/reference/new_forecast.html b/dev/reference/new_forecast.html index a9698e8df..506ce3311 100644 --- a/dev/reference/new_forecast.html +++ b/dev/reference/new_forecast.html @@ -63,8 +63,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-classname">classname<a class="anchor" aria-label="anchor" href="#arg-classname"></a></dt> diff --git a/dev/reference/print.forecast.html b/dev/reference/print.forecast.html index 0c6bebff1..9d2a8b593 100644 --- a/dev/reference/print.forecast.html +++ b/dev/reference/print.forecast.html @@ -60,8 +60,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-x">x<a class="anchor" aria-label="anchor" href="#arg-x"></a></dt> -<dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +<dd><p>A forecast object</p></dd> <dt id="arg--">...<a class="anchor" aria-label="anchor" href="#arg--"></a></dt> diff --git a/dev/reference/quantile_to_interval.html b/dev/reference/quantile_to_interval.html index f51fd66f7..ba352a305 100644 --- a/dev/reference/quantile_to_interval.html +++ b/dev/reference/quantile_to_interval.html @@ -121,7 +121,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A data.table with forecasts in a quantile-based format (see -<code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +<code><a href="as_forecast_quantile.html">as_forecast_quantile()</a></code>).</p></dd> <dt id="arg-format">format<a class="anchor" aria-label="anchor" href="#arg-format"></a></dt> diff --git a/dev/reference/sample_to_interval_long.html b/dev/reference/sample_to_interval_long.html index 0707b388f..3ef3dc9d9 100644 --- a/dev/reference/sample_to_interval_long.html +++ b/dev/reference/sample_to_interval_long.html @@ -62,8 +62,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-type">type<a class="anchor" aria-label="anchor" href="#arg-type"></a></dt> diff --git a/dev/reference/score.html b/dev/reference/score.html index b465b4fbf..2b9f3fdd3 100644 --- a/dev/reference/score.html +++ b/dev/reference/score.html @@ -1,17 +1,21 @@ <!DOCTYPE html> <!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><title>Evaluate forecasts — score.forecast_binary • scoringutils</title><script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet"><script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><link href="../deps/font-awesome-6.4.2/css/all.min.css" rel="stylesheet"><link href="../deps/font-awesome-6.4.2/css/v4-shims.min.css" rel="stylesheet"><script src="../deps/headroom-0.11.0/headroom.min.js"></script><script src="../deps/headroom-0.11.0/jQuery.headroom.min.js"></script><script src="../deps/bootstrap-toc-1.0.1/bootstrap-toc.min.js"></script><script src="../deps/clipboard.js-2.0.11/clipboard.min.js"></script><script src="../deps/search-1.0.0/autocomplete.jquery.min.js"></script><script src="../deps/search-1.0.0/fuse.min.js"></script><script src="../deps/search-1.0.0/mark.min.js"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Evaluate forecasts — score.forecast_binary"><meta name="description" content="score() applies a selection of scoring metrics to a forecast -object (a data.table with forecasts and observations) (see as_forecast()). +object. score() is a generic that dispatches to different methods depending on the class of the input data. -See the Forecast types and input formats section for more information on -forecast types and input formats. +See as_forecast_binary(), as_forecast_quantile() etc. for information on +how to create a forecast object. +See get_forecast_unit() for more information on the concept of a forecast +unit. For additional help and examples, check out the Getting Started Vignette as well as the paper Evaluating Forecasts with scoringutils in R."><meta property="og:description" content="score() applies a selection of scoring metrics to a forecast -object (a data.table with forecasts and observations) (see as_forecast()). +object. score() is a generic that dispatches to different methods depending on the class of the input data. -See the Forecast types and input formats section for more information on -forecast types and input formats. +See as_forecast_binary(), as_forecast_quantile() etc. for information on +how to create a forecast object. +See get_forecast_unit() for more information on the concept of a forecast +unit. For additional help and examples, check out the Getting Started Vignette as well as the paper Evaluating Forecasts with scoringutils in R."><meta name="robots" content="noindex"></head><body> <a href="#main" class="visually-hidden-focusable">Skip to contents</a> @@ -55,12 +59,14 @@ <h1>Evaluate forecasts</h1> <div class="ref-description section level2"> <p><code>score()</code> applies a selection of scoring metrics to a forecast -object (a data.table with forecasts and observations) (see <code><a href="as_forecast.html">as_forecast()</a></code>). +object. <code>score()</code> is a generic that dispatches to different methods depending on the class of the input data.</p> -<p>See the <em>Forecast types and input formats</em> section for more information on -forecast types and input formats. -For additional help and examples, check out the <a href="https://epiforecasts.io/scoringutils/articles/scoringutils.html">Getting Started Vignette</a> as +<p>See <code><a href="as_forecast_binary.html">as_forecast_binary()</a></code>, <code><a href="as_forecast_quantile.html">as_forecast_quantile()</a></code> etc. for information on +how to create a forecast object.</p> +<p>See <code><a href="get_forecast_unit.html">get_forecast_unit()</a></code> for more information on the concept of a forecast +unit.</p> +<p>For additional help and examples, check out the <a href="https://epiforecasts.io/scoringutils/articles/scoringutils.html">Getting Started Vignette</a> as well as the paper <a href="https://arxiv.org/abs/2205.07090" class="external-link">Evaluating Forecasts with scoringutils in R</a>.</p> </div> @@ -90,7 +96,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-metrics">metrics<a class="anchor" aria-label="anchor" href="#arg-metrics"></a></dt> @@ -135,69 +141,6 @@ <h2 id="details">Details<a class="anchor" aria-label="anchor" href="#details"></ <span id="cb1-5"><a href="#cb1-5" tabindex="-1"></a>custom_arg <span class="ot"><-</span> <span class="st">"bar"</span></span> <span id="cb1-6"><a href="#cb1-6" tabindex="-1"></a><span class="fu">print1</span>() <span class="co"># prints 'bar'</span></span> <span id="cb1-7"><a href="#cb1-7" tabindex="-1"></a><span class="fu">print2</span>() <span class="co"># prints 'foo'</span></span></code></pre><p></p></div> - </div> - <div class="section level2"> - <h2 id="forecast-types-and-input-formats">Forecast types and input formats<a class="anchor" aria-label="anchor" href="#forecast-types-and-input-formats"></a></h2> - <p>Various different forecast types / forecast formats are supported. At the -moment, those are:</p><ul><li><p>point forecasts</p></li> -<li><p>binary forecasts ("soft binary classification")</p></li> -<li><p>nominal forecasts ("soft classification with multiple unordered classes")</p></li> -<li><p>Probabilistic forecasts in a quantile-based format (a forecast is -represented as a set of predictive quantiles)</p></li> -<li><p>Probabilistic forecasts in a sample-based format (a forecast is represented -as a set of predictive samples)</p></li> -</ul><p>Forecast types are determined based on the columns present in the input data. -Here is an overview of the required format for each forecast type: - - </p><div style="text-align: left"> - <img src="figures/required-inputs.png" style="width:750px;max-width:100%;"></div> -<p><em>All forecast types</em> require a data.frame or similar with columns <code>observed</code> -<code>predicted</code>, and <code>model</code>.</p> -<p><em>Point forecasts</em> require a column <code>observed</code> of type numeric and a column -<code>predicted</code> of type numeric.</p> -<p><em>Binary forecasts</em> require a column <code>observed</code> of type factor with exactly -two levels and a column <code>predicted</code> of type numeric with probabilities, -corresponding to the probability that <code>observed</code> is equal to the second -factor level. See details <a href="scoring-functions-binary.html">here</a> for more information.</p> -<p><em>Nominal forecasts</em> require a column <code>observed</code> of type factor with N levels, -(where N is the number of possible outcomes), a column <code>predicted</code> of type -numeric with probabilities (which sum to one across all possible outcomes), -and a column <code>predicted_label</code> of type factor with N levels, denoting the -outcome for which a probability is given. Forecasts must be complete, i.e. -there must be a probability assigned to every possible outcome.</p> -<p><em>Quantile-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>quantile_level</code> of type -numeric with quantile-levels (between 0 and 1).</p> -<p><em>Sample-based forecasts</em> require a column <code>observed</code> of type numeric, -a column <code>predicted</code> of type numeric, and a column <code>sample_id</code> of type -numeric with sample indices.</p> -<p>For more information see the vignettes and the example data -(<a href="example_quantile.html">example_quantile</a>, <a href="example_sample_continuous.html">example_sample_continuous</a>, <a href="example_sample_discrete.html">example_sample_discrete</a>, -<code><a href="example_point.html">example_point()</a></code>, <a href="example_binary.html">example_binary</a>, and <a href="example_nominal.html">example_nominal</a>).</p> - </div> - <div class="section level2"> - <h2 id="forecast-unit">Forecast unit<a class="anchor" aria-label="anchor" href="#forecast-unit"></a></h2> - <p>In order to score forecasts, <code>scoringutils</code> needs to know which of the rows -of the data belong together and jointly form a single forecasts. This is -easy e.g. for point forecast, where there is one row per forecast. For -quantile or sample-based forecasts, however, there are multiple rows that -belong to a single forecast.</p> -<p>The <em>forecast unit</em> or <em>unit of a single forecast</em> is then described by the -combination of columns that uniquely identify a single forecast. -For example, we could have forecasts made by different models in various -locations at different time points, each for several weeks into the future. -The forecast unit could then be described as -<code>forecast_unit = c("model", "location", "forecast_date", "forecast_horizon")</code>. -<code>scoringutils</code> automatically tries to determine the unit of a single -forecast. It uses all existing columns for this, which means that no columns -must be present that are unrelated to the forecast unit. As a very simplistic -example, if you had an additional row, "even", that is one if the row number -is even and zero otherwise, then this would mess up scoring as <code>scoringutils</code> -then thinks that this column was relevant in defining the forecast unit.</p> -<p>In order to avoid issues, we recommend setting the forecast unit explicitly, -usually through the <code>forecast_unit</code> argument in the <code><a href="as_forecast.html">as_forecast()</a></code> -functions. This will drop unneeded columns, while making sure that all -necessary, 'protected columns' like "predicted" or "observed" are retained.</p> </div> <div class="section level2"> <h2 id="references">References<a class="anchor" aria-label="anchor" href="#references"></a></h2> diff --git a/dev/reference/scoring-functions-nominal.html b/dev/reference/scoring-functions-nominal.html index 984954ee2..988c31b05 100644 --- a/dev/reference/scoring-functions-nominal.html +++ b/dev/reference/scoring-functions-nominal.html @@ -80,6 +80,11 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <div class="section level2"> <h2 id="value">Value<a class="anchor" aria-label="anchor" href="#value"></a></h2> <p>A numeric vector of size n with log scores</p> + </div> + <div class="section level2"> + <h2 id="input-format">Input format<a class="anchor" aria-label="anchor" href="#input-format"></a></h2> + <p></p><div style="text-align: left"> + <img src="figures/metrics-nominal.png" style="width:750px;max-width:100%;"></div> </div> <div class="section level2"> <h2 id="see-also">See also<a class="anchor" aria-label="anchor" href="#see-also"></a></h2> diff --git a/dev/reference/select_metrics.html b/dev/reference/select_metrics.html index 6bc3e85ac..ca1421886 100644 --- a/dev/reference/select_metrics.html +++ b/dev/reference/select_metrics.html @@ -88,7 +88,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> brierscore <- (observed - predicted)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(brierscore)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba2740></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc441010></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> <span class="r-in"><span><span class="fu">select_metrics</span><span class="op">(</span></span></span> @@ -103,7 +103,7 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e <span class="r-out co"><span class="r-pr">#></span> brierscore <- (observed - predicted)^2</span> <span class="r-out co"><span class="r-pr">#></span> return(brierscore)</span> <span class="r-out co"><span class="r-pr">#></span> }</span> -<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x55dc54ba2740></span> +<span class="r-out co"><span class="r-pr">#></span> <bytecode: 0x5626cc441010></span> <span class="r-out co"><span class="r-pr">#></span> <environment: namespace:scoringutils></span> <span class="r-out co"><span class="r-pr">#></span> </span> </code></pre></div> diff --git a/dev/reference/set_forecast_unit.html b/dev/reference/set_forecast_unit.html index d5e8be4b2..ca1590aa8 100644 --- a/dev/reference/set_forecast_unit.html +++ b/dev/reference/set_forecast_unit.html @@ -4,8 +4,8 @@ This simple function keeps the columns specified in forecast_unit (plus additional protected columns, e.g. for observed values, predictions or quantile levels) and removes duplicate rows. set_forecast_unit() will -mainly be called when constructing a forecast object (see as_forecast()) -via the forecast_unit argument there. +mainly be called when constructing a forecast object +via the forecast_unit argument in as_forecast_&lt;type&gt;. If not done explicitly, scoringutils attempts to determine the unit of a single forecast automatically by simply assuming that all column names are relevant to determine the forecast unit. This may lead to unexpected @@ -15,8 +15,8 @@ This simple function keeps the columns specified in forecast_unit (plus additional protected columns, e.g. for observed values, predictions or quantile levels) and removes duplicate rows. set_forecast_unit() will -mainly be called when constructing a forecast object (see as_forecast()) -via the forecast_unit argument there. +mainly be called when constructing a forecast object +via the forecast_unit argument in as_forecast_&lt;type&gt;. If not done explicitly, scoringutils attempts to determine the unit of a single forecast automatically by simply assuming that all column names are relevant to determine the forecast unit. This may lead to unexpected @@ -67,8 +67,8 @@ <h1>Set unit of a single forecast manually</h1> This simple function keeps the columns specified in <code>forecast_unit</code> (plus additional protected columns, e.g. for observed values, predictions or quantile levels) and removes duplicate rows. <code>set_forecast_unit()</code> will -mainly be called when constructing a <code>forecast</code> object (see <code><a href="as_forecast.html">as_forecast()</a></code>) -via the <code>forecast_unit</code> argument there.</p> +mainly be called when constructing a <code>forecast</code> object +via the <code>forecast_unit</code> argument in <code>as_forecast_<type></code>.</p> <p>If not done explicitly, <code>scoringutils</code> attempts to determine the unit of a single forecast automatically by simply assuming that all column names are relevant to determine the forecast unit. This may lead to unexpected @@ -87,8 +87,8 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-data">data<a class="anchor" aria-label="anchor" href="#arg-data"></a></dt> <dd><p>A data.frame (or similar) with predicted and observed values. -See the details section of <code><a href="as_forecast.html">as_forecast()</a></code> for additional information -on required input formats.</p></dd> +See the details section of for additional information +on the required input format.</p></dd> <dt id="arg-forecast-unit">forecast_unit<a class="anchor" aria-label="anchor" href="#arg-forecast-unit"></a></dt> diff --git a/dev/reference/transform_forecasts.html b/dev/reference/transform_forecasts.html index 371a0fb31..4ed7026f0 100644 --- a/dev/reference/transform_forecasts.html +++ b/dev/reference/transform_forecasts.html @@ -60,7 +60,7 @@ <h2 id="arguments">Arguments<a class="anchor" aria-label="anchor" href="#argumen <dl><dt id="arg-forecast">forecast<a class="anchor" aria-label="anchor" href="#arg-forecast"></a></dt> <dd><p>A forecast object (a validated data.table with predicted and -observed values, see <code><a href="as_forecast.html">as_forecast()</a></code>).</p></dd> +observed values).</p></dd> <dt id="arg-fun">fun<a class="anchor" aria-label="anchor" href="#arg-fun"></a></dt> diff --git a/dev/search.json b/dev/search.json index cadbee6e2..8e5093aca 100644 --- a/dev/search.json +++ b/dev/search.json @@ -1 +1 @@ -[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @returns a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) library(ggdist) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (\"quantile_level\" %in% colnames(data)) { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (\"sample_id\" %in% colnames(data)) { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast_sample(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast_quantile(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast_quantile() %>% score(metrics = list(wis = wis, dispersion = dispersion_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @returns A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast_quantile(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast_quantile(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form S(ŷ,y)S(\\hat{y}, y), ŷ\\hat{y} forecast yy observation. probabilistic forecasts, usually take form S(F,y)S(F, y), FF cumulative density function (CDF) predictive distribution yy observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let GG true, unobserved data-generating distribution. scoring rule said proper, GG ideal forecast F=GF = G, forecast F′≠FF' \\neq F expectation receives better score FF. scoring rule considered strictly proper , GG, forecast F′F' expectation receives score better FF.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling get_metrics(example_point). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. ae=|y−ŷ|\\text{ae} = |y - \\hat{y}| absolute error appropriate rule ŷ\\hat{y} corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. se=(y−ŷ)2\\text{se} = (y - \\hat{y})^2 squared error appropriate rule ŷ\\hat{y} corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. ape=|y−ŷ||y|\\text{ape} = \\frac{|y - \\hat{y}|}{|y|} absolute percentage error appropriate rule ŷ\\hat{y} corresponds β\\beta-median forecaster’s predictive distribution β=−1\\beta = -1. β\\beta-median, med(β)(F)\\text{med}^{(\\beta)}(F), median random variable whose density proportional yβf(y)y^\\beta f(y). specific β\\beta-median corresponds absolute percentage error med(−1)(F)\\text{med}^{(-1)}(F). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?get_metrics(example_binary). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. BS(p,y)=(p−y)2={p2,y=1(1−p)2,y=0\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation} Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. Log score(p,y)=−log(1−|y−p|)={−log(p),y=1−log(1−p),y=0\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation} log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling get_metrics(example_sample_continuous). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined CRPS(F,y)=∫−∞∞(F(x)−1(x≥y))2dx,\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx, yy observed value FF CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : RPS(P,y)=∑x=0∞(P(x)−1(x≥y))2, \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, PP cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Sample-based forecasts > CRPS","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"CRPS can interpreted sum dispersion, overprediction underprediction component. mm median forecast dispersion component CRPS(F,m),\\text{CRPS}(F, m), overprediction component {m>yCRPS(F,y)−CRPS(F,m)m≤y0 \\begin{cases} m > y & CRPS(F, y) - CRPS(F, m)\\\\ m \\leq y & 0\\\\ \\end{cases} underprediction component {m<yCRPS(F,y)−CRPS(F,m)m≥y0 \\begin{cases} m < y & CRPS(F, y) - CRPS(F, m)\\\\ m \\geq y & 0\\\\ \\end{cases} can accessed via dispersion_sample(), overprediction_sample() underprediction_sample() functions, respectively.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"log-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Log score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. logarithmic scoring rule simply negative logarithm density predictive distribution evaluated observed value: log score(F,y)=−logf(y), \\text{log score}(F, y) = -\\log f(y), ff predictive probability density function (PDF) corresponding Forecast FF yy observed value. discrete forecasts, log score can computed log score(F,y)=−logpy, \\text{log score}(F, y) = -\\log p_y, pyp_y probability assigned observed outcome yy forecast FF. logarithmic scoring rule can produce large penalties observed value takes values f(y)f(y) (pyp_y) close zero. therefore considered sensitive outlier forecasts. may desirable applications, also means scores can easily dominated extreme values. logarithmic scoring rule local scoring rule, meaning score depends probability assigned actual outcome. often regarded desirable property example context Bayesian inference . implies example, ranking forecasters invariant monotone transformations predictive distribution target. scoringutils re-exports logs_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. One implications currently advisable use log score discrete forecasts. reason scoringRules::logs_sample() estimates predictive density samples, can problematic discrete forecasts. See ?logs_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dawid-sebastiani-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dawid-Sebastiani score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: FF. predictive distribution mean μ\\mu standard deviation σ\\sigma. Dawid-Sebastiani score proper scoring rule relies first moments predictive distribution therefore easy compute. given dss(F,y)=(y−μσ)2+2⋅logσ.\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma. scoringutils re-exports implementation DSS scoringRules package. assumes forecast represented set samples drawn predictive distribution. See ?dss_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dispersion---median-absolute-deviation-mad","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dispersion - Median Absolute Deviation (MAD)","title":"Scoring rules in `scoringutils`","text":"Observation: required. Forecast: FF, predictive distribution. Dispersion (also called sharpness) ability produce narrow forecasts. feature forecasts depend observations. Dispersion therefore interest conditional calibration: precise forecast useful clearly wrong. One way measure sharpness (suggested Funk et al. (2019)) normalised median absolute deviation median (MADN) ). computed S(F)=10.675⋅median(|F−median(F)|). S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|F - \\text{median(F)}|). forecast FF follows normal distribution, sharpness equal standard deviation FF.details, see ?mad_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. Bias measure tendency forecaster - underpredict. continuous forecasts, scoringutils implementation calculates bias B(F,y)=1−2⋅F(y),B(F, y) = 1 - 2 \\cdot F (y), F(y)F(y) cumulative distribution function forecast. discrete forecasts, calculate bias B(P,y)=1−(P(y)+P(y+1)).B(P, y) = 1 - (P(y) + P(y + 1)). P(y)P(y) cumulative probability assigned outcomes smaller equal yy, .e. cumulative probability mass function. Bias bound -1 1 represents tendency forecasts biased rather absolute amount - underprediction (e.g. case weighted interval score (WIS, see ).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF aemedian(F,y)=|median(F)−y|.\\text{ae}_{\\text{median}}(F, y) = |\\text{median} (F) - y|. See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error-of-the-mean","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Squared error of the mean","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF semedn(F,y)=(mean(F)−y)2.\\text{se}_{\\text{medn}}(F, y) = (\\text{mean} (F) - y)^2. See section note caution Gneiting (2011) discussion correspondence squared error mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Quantile-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics quantile-based forecasts calling get_metrics(example_quantile). overview input output formats quantile forecasts: Input output formats: metrics quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"weighted-interval-score-wis","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Weighted interval score (WIS)","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form lower (ll) upper (uu) bounds central prediction intervals. weighted interval score (WIS) strictly proper scoring rule can understood approximation CRPS forecasts quantile format (turn represents generalisation absolute error). Quantiles assumed lower upper bounds prediction intervals symmetric around median. single interval, interval score ISα(F,y)=(u−l)⏟dispersion+2α⋅(l−y)⋅𝟏(y≤l)⏟overprediction+2α⋅(y−u)⋅𝟏(y≥u)⏟underprediction,IS_\\alpha(F,y) = \\underbrace{(u-l)}_\\text{dispersion} + \\underbrace{\\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l)}_{\\text{overprediction}} + \\underbrace{\\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u)}_{\\text{underprediction}}, 𝟏()\\mathbf{1}() indicator function, ll uu α2\\frac{\\alpha}{2} 1−α21 - \\frac{\\alpha}{2} quantiles predictive distribution FF. ll uu together form prediction interval. interval score can understood sum three components: dispersion, overprediction underprediction. set KK prediction intervals median mm, score given weighted sum individual interval scores, .e. WIS=1K+0.5⋅(w0⋅|y−m|+∑k=1Kwk⋅ISαk(F,y)),WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha_{k}}(F, y)\\right), mm median forecast wkw_k weight assigned every interval. weights set wk=αk2w_k = \\frac{\\alpha_k}{2} w0=0.5w_0 = 0.5, WIS converges CRPS increasing number equally spaced quantiles. See ?wis() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion-1","dir":"Articles","previous_headings":"Quantile-based forecasts > Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction_quantile(), ?underprediction_quantile() ?dispersion_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles, QQ. Bias can measured B(F,y)={(1−2⋅max{α|qα∈Q∧qα≤y}),y<q0.5(overprediction)(1−2⋅min{α|qα∈Qt∧qα≥y},y>q0.5(underprediction)0,y=q0.5,\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation} qαq_\\alpha α\\alpha-quantile predictive distribution. consistency, define QQ (set quantiles form predictive distribution FF) always includes element q0=−∞q_0 = -\\infty q1=∞q_1 = \\infty. clearer terms, bias : 1−(2×1 - (2 \\times maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. 1−(2×1 - (2 \\times minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. 00if observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation 0.6=−0.5=0.1.0.6 = - 0.5 = 0.1. interval coverage deviation=observed interval coverage−nominal interval coverage\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. aemedian=|median(F)−y|\\text{ae}_\\text{median} = |\\text{median}(F) - y| See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level τ\\tau defined QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,y≤qτ2⋅τ*|qτ−y|,y>qτ,\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation} qτq_\\tau τ\\tau-quantile predictive distribution FF, 𝟏(⋅)\\mathbf{1}(\\cdot) indicator function. (unweighted) interval score (see ) 1−α1 - \\alpha prediction interval can computed quantile scores levels α/2\\alpha/2 1−α/21 - \\alpha/2 ISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)α\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}. weighted interval score can obtained simple average quantile scores: WISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)2\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author. Michael Chirico. Contributor.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast_quantile( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ) ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted location forecast_date target_end_date #> <num> <num> <int> <char> <Date> <Date> #> 1: 127300 NA NA DE <NA> 2021-01-02 #> 2: 4534 NA NA DE <NA> 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> <char> <char> <num> #> 1: Cases <NA> NA #> 2: Deaths <NA> NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast_<type>() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> <num> <num> <num> <num> <num> #> 1: 3660.0 -0.05640 0.391 0.805 24100.0 #> 2: 4100.0 0.09800 0.328 0.820 38500.0 #> 3: 5660.0 -0.07890 0.469 0.789 27900.0 #> 4: 30.2 0.07270 0.875 1.000 53.1 #> 5: 91.4 0.33900 0.664 1.000 233.0 #> 6: 26.9 -0.02230 0.461 0.875 78.5 #> 7: 31.9 -0.00513 0.420 0.908 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(<citation>, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"funding","dir":"","previous_headings":"","what":"Funding","title":"Utilities for Scoring and Assessing Predictions","text":"development scoringutils funded via Health Protection Research Unit (grant code NIHR200908) Wellcome Trust (grant: 210758/Z/18/Z). work also supported US National Institutes General Medical Sciences (R35GM119582). content solely responsibility authors necessarily represent official views NIGMS, National Institutes Health.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributors","dir":"","previous_headings":"","what":"Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"contributions project gratefully acknowledged using allcontributors package following -contributors specification. Contributions kind welcome!","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code","dir":"","previous_headings":"Contributors","what":"Code","title":"Utilities for Scoring and Assessing Predictions","text":"nikosbosse, seabbs, jamesmbaazam, Bisaloo, sbfnk, actions-user, toshiakiasakura, MichaelChirico, nickreich, jhellewell14, damonbayer","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-authors","dir":"","previous_headings":"Contributors","what":"Issue Authors","title":"Utilities for Scoring and Assessing Predictions","text":"DavideMagno, mbojan, dshemetov, elray1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-contributors","dir":"","previous_headings":"Contributors","what":"Issue Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"jbracher, dylanhmorris, jonathonmellor, kathsherratt","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ |observed - median prediction| $$ median prediction predicted value quantile_level == 0.5. function requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$ |observed - median\\_prediction| $$ median prediction calculated median predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Numeric vector length n absolute errors median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using purrr::partial().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"General information on creating a forecast object — as_forecast","title":"General information on creating a forecast object — as_forecast","text":"several as_forecast_<type>() functions process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. Every forecast type as_forecast_<type>() function. See details section information expected input formats. as_forecast_<type>() functions give users control data parsed. Using arguments observed, predicted, etc. users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see docs internal set_forecast_unit() details). following functions available: as_forecast_point() as_forecast_binary() as_forecast_sample() as_forecast_quantile()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General information on creating a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"General information on creating a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"General information on creating a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"General information on creating a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"General information on creating a forecast object — as_forecast","text":"","code":"as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> <char> <char> <Date> <char> <Date> #> 1: DE Germany 2021-01-02 Cases <NA> #> 2: DE Germany 2021-01-02 Deaths <NA> #> 3: DE Germany 2021-01-09 Cases <NA> #> 4: DE Germany 2021-01-09 Deaths <NA> #> 5: DE Germany 2021-01-16 Cases <NA> #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> <char> <num> <num> <fctr> #> 1: <NA> NA NA <NA> #> 2: <NA> NA NA <NA> #> 3: <NA> NA NA <NA> #> 4: <NA> NA NA <NA> #> 5: <NA> NA NA <NA> #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast_quantile( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted model target_type #> <num> <num> <int> <char> <char> #> 1: 127300 NA NA <NA> Cases #> 2: 4534 NA NA <NA> Deaths #> 3: 154922 NA NA <NA> Cases #> 4: 6117 NA NA <NA> Deaths #> 5: 110183 NA NA <NA> Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> <Date> <num> <char> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for binary forecasts — as_forecast_binary","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"Create forecast object binary forecasts. See information forecast types expected input formats calling ?as_forecast().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"","code":"as_forecast_binary( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Common functionality for as_forecast_<type> functions — as_forecast_generic","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"Common functionality as_forecast_<type> functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"","code":"as_forecast_generic( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"function splits part functionality as_forecast_<type> as_forecast_<type> functions. renames required columns, appropriate, sets forecast unit.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for nominal forecasts — as_forecast_nominal","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"Nominal forecasts form categorical forecasts possible outcomes observed values can assume ordered. sense, Nominal forecasts represent generalisation binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"","code":"as_forecast_nominal( data, forecast_unit = NULL, observed = NULL, predicted = NULL, predicted_label = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". predicted_label (optional) Name column data denotes outcome predicted probability corresponds . column renamed \"predicted_label\". applicable nominal forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for point forecasts — as_forecast_point","title":"Create a forecast object for point forecasts — as_forecast_point","text":"Create forecast object point forecasts. See information forecast types expected input formats calling ?as_forecast(). converting forecast_quantile object forecast_point object, 0.5 quantile extracted returned point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for point forecasts — as_forecast_point","text":"","code":"as_forecast_point(data, ...) # Default S3 method as_forecast_point( data, forecast_unit = NULL, observed = NULL, predicted = NULL, ... ) # S3 method for class 'forecast_quantile' as_forecast_point(data, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for point forecasts — as_forecast_point","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"Create forecast object quantile-based forecasts. See information forecast types expected input formats calling ?as_forecast(). creating forecast_quantile object forecast_sample object, quantiles estimated computing empircal quantiles samples via quantile(). Note empirical quantiles biased estimator true quantiles particular tails distribution number available samples low.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"","code":"as_forecast_quantile(data, ...) # Default S3 method as_forecast_quantile( data, forecast_unit = NULL, observed = NULL, predicted = NULL, quantile_level = NULL, ... ) # S3 method for class 'forecast_sample' as_forecast_quantile( data, probs = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. probs numeric vector quantile levels quantiles computed. Corresponds probs argument quantile(). type Type argument passed quantile function. information, see quantile().","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for sample-based forecasts — as_forecast_sample","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"Create forecast object sample-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"","code":"as_forecast_sample( data, forecast_unit = NULL, observed = NULL, predicted = NULL, sample_id = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Assert object forecast object (.e. data.table class forecast additional class forecast_* corresponding forecast type).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Returns NULL invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"forecast <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object (see as_forecast()). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Function assesses whether inputs correspond requirements scoring nominal forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"","code":"assert_input_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"observed Input checked. factor length n N levels holding observed values. n number observations N number possible outcomes observed values can assume. output) predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level. predicted_label Factor length N N levels, N number possible outcomes observed values can assume.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — assert_scores","title":"Validate an object of class scores — assert_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — assert_scores","text":"","code":"assert_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — assert_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — assert_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed linear interpolation two innermost quantiles. median available imputed, error thrown. Note order compute bias, quantiles must non-decreasing increasing quantile levels. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of rows — check_number_per_forecast","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Helper function checks number rows (corresponding e.g quantiles samples) per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare a subset of common forecasts — compare_forecasts","title":"Compare a subset of common forecasts — compare_forecasts","text":"function compares two comparators based subset forecasts comparators made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple comparators single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare a subset of common forecasts — compare_forecasts","text":"","code":"compare_forecasts( scores, compare = \"model\", name_comparator1, name_comparator2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare a subset of common forecasts — compare_forecasts","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. name_comparator1 Character, name first comparator name_comparator2 Character, name comparator compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare a subset of common forecasts — compare_forecasts","text":"list mean score ratios p-values comparison two comparators","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare a subset of common forecasts — compare_forecasts","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts Continuous ranked probability score (CRPS) can interpreted sum three components: overprediction, underprediction dispersion. \"Dispersion\" defined CRPS median forecast $m$. observation $y$ greater $m$ overpredictoin defined CRPS forecast $y$ minus dispersion component, underprediction zero. , hand, $y<m$ underprediction defined CRPS forecast $y$ minus dispersion component, overprediction zero. overprediction, underprediction dispersion components correspond wis().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"crps_sample(observed, predicted, separate_results = FALSE, ...) dispersion_sample(observed, predicted, ...) overprediction_sample(observed, predicted, ...) underprediction_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"(Continuous) ranked probability score — crps_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. separate_results Logical. TRUE (default FALSE), separate parts CRPS (dispersion penalty, penalties - -prediction) get returned separate elements list. want data.frame instead, simply call .data.frame() output. ... Additional arguments passed crps_sample() functions overprediction_sample(), underprediction_sample() dispersion_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"(Continuous) ranked probability score — crps_sample","text":"Vector scores. dispersion_sample(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Continuous) ranked probability score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(observed, predicted) #> [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"object class forecast_binary (see as_forecast()) following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Nominal example data — example_nominal","title":"Nominal example data — example_nominal","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Nominal example data — example_nominal","text":"","code":"example_nominal"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Nominal example data — example_nominal","text":"object class forecast_nominal (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made predicted_label outcome probabilty corresponds predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Nominal example data — example_nominal","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Nominal example data — example_nominal","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"object class forecast_point (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"object class forecast_quantile (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics.scores(scores), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores) #> wis overprediction underprediction dispersion bias #> <num> <num> <num> <num> <num> #> 1: 1.0000000 0.94297565 0.28377361 0.45566303 0.10545891 #> 2: 0.9429757 1.00000000 -0.03310356 0.32493799 0.21532161 #> 3: 0.2837736 -0.03310356 1.00000000 0.14580143 -0.35123801 #> 4: 0.4556630 0.32493799 0.14580143 1.00000000 0.11118365 #> 5: 0.1054589 0.21532161 -0.35123801 0.11118365 1.00000000 #> 6: -0.2076649 -0.14556039 -0.21392764 -0.09400664 0.01338140 #> 7: -0.4075613 -0.31824017 -0.35756699 -0.08614678 0.09802725 #> 8: 0.9886108 0.90326672 0.33589892 0.53809741 0.09578751 #> interval_coverage_50 interval_coverage_90 ae_median metric #> <num> <num> <num> <char> #> 1: -0.20766492 -0.40756133 0.98861080 wis #> 2: -0.14556039 -0.31824017 0.90326672 overprediction #> 3: -0.21392764 -0.35756699 0.33589892 underprediction #> 4: -0.09400664 -0.08614678 0.53809741 dispersion #> 5: 0.01338140 0.09802725 0.09578751 bias #> 6: 1.00000000 0.37245118 -0.24559356 interval_coverage_50 #> 7: 0.37245118 1.00000000 -0.41079097 interval_coverage_90 #> 8: -0.24559356 -0.41079097 1.00000000 ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> <char> <num> <num> <num> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> <num> <num> <num> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Internal helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data, forecast_unit = NULL, counts = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. counts output show number duplicates per forecast unit instead individual duplicated rows? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 5: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 6: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 13: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 14: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> <num> <int> <char> <num> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.950 1464 epiforecasts-EpiNow2 1 #> 3: 0.975 1642 epiforecasts-EpiNow2 1 #> 4: 0.975 1642 epiforecasts-EpiNow2 1 #> 5: 0.990 1951 epiforecasts-EpiNow2 1 #> 6: 0.990 1951 epiforecasts-EpiNow2 1 #> 7: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 8: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 9: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 10: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 11: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 12: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 13: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 14: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 15: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 16: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 17: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 18: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 19: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 20: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 21: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_forecast_counts(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: <model, target_type> #> model target_type count #> <char> <char> <int> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get forecast type from forecast object — get_forecast_type","title":"Get forecast type from forecast object — get_forecast_type","text":"Get forecast type forecast object","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get forecast type from forecast object — get_forecast_type","text":"","code":"get_forecast_type(forecast)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get forecast type from forecast object — get_forecast_type","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get forecast type from forecast object — get_forecast_type","text":"Character vector length one forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for binary forecasts — get_metrics.forecast_binary","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"binary forecasts, default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"list scoring functions.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"get_metrics(example_binary) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x55dc54ba2740> #> <environment: namespace:scoringutils> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> <bytecode: 0x55dc54ba1908> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x55dc54ba2740> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x55dc54ba2740> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"nominal forecasts, default scoring rule : \"log_score\" = logs_nominal()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"# S3 method for class 'forecast_nominal' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"get_metrics(example_nominal) #> $log_score #> function (observed, predicted, predicted_label) #> { #> assert_input_nominal(observed, predicted, predicted_label) #> n <- length(observed) #> if (n == 1) { #> predicted <- matrix(predicted, nrow = 1) #> } #> observed_indices <- as.numeric(observed) #> pred_for_observed <- predicted[cbind(1:n, observed_indices)] #> logs <- -log(pred_for_observed) #> return(logs) #> } #> <bytecode: 0x55dc52980ec8> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for point forecasts — get_metrics.forecast_point","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"point forecasts, default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape() note caution: Every scoring rule point forecast implicitly minimised specific aspect predictive distribution (see Gneiting, 2011). mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, results misleading. Failure respect correspondence can lead grossly misleading results! Consider example section .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"# S3 method for class 'forecast_point' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"Making Evaluating Point Forecasts, Gneiting, Tilmann, 2011, Journal American Statistical Association.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"get_metrics(example_point, select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> <bytecode: 0x55dc550ceb38> #> <environment: namespace:Metrics> #> library(magrittr) set.seed(123) n <- 500 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) df <- data.frame( model = rep(c(\"perfect\", \"bad\"), each = n), predicted = c(rep(predicted_mu, n), predicted_not_mu), observed = rep(observed, 2), id = rep(1:n, 2) ) %>% as_forecast_point() score(df) %>% summarise_scores() #> model ae_point se_point ape #> <char> <num> <num> <num> #> 1: perfect 34.64686 2145.813 3543.184 #> 2: bad 32.34199 2238.566 2692.868"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"quantile-based forecasts, default scoring rules : \"wis\" = wis() \"overprediction\" = overprediction_quantile() \"underprediction\" = underprediction_quantile() \"dispersion\" = dispersion_quantile() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = purrr::partial( interval_coverage, interval_range = 90 ) \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function purrr::partial(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"get_metrics(example_quantile, select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> interval_ranges <- get_range_from_quantile(quantile_level[quantile_level != #> 0.5]) #> complete_intervals <- duplicated(interval_ranges) | duplicated(interval_ranges, #> fromLast = TRUE) #> if (!all(complete_intervals) && !isTRUE(na.rm)) { #> incomplete <- quantile_level[quantile_level != 0.5][!complete_intervals] #> cli_abort(c(`!` = \"Not all quantile levels specified form symmetric prediction\\n intervals.\\n The following quantile levels miss a corresponding lower/upper bound:\\n {.val {incomplete}}.\\n You can drop incomplete prediction intervals using `na.rm = TRUE`.\")) #> } #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> <bytecode: 0x55dc56d4c188> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"sample-based forecasts, default scoring rules : \"crps\" = crps_sample() \"overprediction\" = overprediction_sample() \"underprediction\" = underprediction_sample() \"dispersion\" = dispersion_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"# S3 method for class 'forecast_sample' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"get_metrics(example_sample_continuous, exclude = \"mad\") #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> <bytecode: 0x55dc57067b00> #> <environment: namespace:scoringutils> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> <bytecode: 0x55dc55ab8df0> #> <environment: namespace:scoringutils> #> #> $crps #> function (observed, predicted, separate_results = FALSE, ...) #> { #> assert_input_sample(observed, predicted) #> crps <- scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> if (separate_results) { #> medians <- apply(predicted, 1, median) #> dispersion <- scoringRules::crps_sample(y = medians, #> dat = predicted, ...) #> overprediction <- rep(0, length(observed)) #> underprediction <- rep(0, length(observed)) #> overprediction[observed < medians] <- scoringRules::crps_sample(y = observed[observed < #> medians], dat = predicted[observed < medians, , drop = FALSE], #> ...) #> underprediction[observed > medians] <- scoringRules::crps_sample(y = observed[observed > #> medians], dat = predicted[observed > medians, , drop = FALSE], #> ...) #> overprediction[overprediction > 0] <- overprediction[overprediction > #> 0] - dispersion[overprediction > 0] #> underprediction[underprediction > 0] <- underprediction[underprediction > #> 0] - dispersion[underprediction > 0] #> return(list(crps = crps, dispersion = dispersion, underprediction = underprediction, #> overprediction = overprediction)) #> } #> else { #> return(crps) #> } #> } #> <bytecode: 0x55dc54553238> #> <environment: namespace:scoringutils> #> #> $overprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$overprediction) #> } #> <bytecode: 0x55dc57e45db8> #> <environment: namespace:scoringutils> #> #> $underprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$underprediction) #> } #> <bytecode: 0x55dc57e490c0> #> <environment: namespace:scoringutils> #> #> $dispersion #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$dispersion) #> } #> <bytecode: 0x55dc57e48528> #> <environment: namespace:scoringutils> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> <bytecode: 0x55dc57e47a00> #> <environment: namespace:scoringutils> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> <bytecode: 0x55dc55b6ea58> #> <environment: namespace:scoringutils> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> <bytecode: 0x55dc57e49f08> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get metrics — get_metrics","title":"Get metrics — get_metrics","text":"Generic function obtain default metrics availble scoring metrics used scoring. called forecast object returns list functions can used scoring. called scores object (see score()), returns character vector names metrics used scoring. See documentation actual methods See Also section details. Alternatively call ?get_metrics.<forecast_type> ?get_metrics.scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get metrics — get_metrics","text":"","code":"get_metrics(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get metrics — get_metrics","text":"x forecast scores object. ... Additional arguments passed method.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Get metrics — get_metrics","text":"See as_forecast() information forecast objects score() information scores objects.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics.scores","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"","code":"# S3 method for class 'scores' get_metrics(x, error = FALSE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"x scores object, (data.table attribute metrics produced score()). error Throw error attribute called metrics? Default FALSE. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"Character vector names scoring rules used scoring.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"Generate Probability Integral Transformation (PIT) histogram validated forecast objects. See examples plot result function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_pit_histogram(forecast, num_bins = NULL, breaks = NULL, by, ...) # S3 method for class 'forecast_sample' get_pit_histogram( forecast, num_bins = 10, breaks = NULL, by, integers = c(\"nonrandom\", \"random\", \"ignore\"), n_replicates = NULL, ... ) get_pit_histogram(forecast, num_bins, breaks, by, ...) # Default S3 method get_pit_histogram(forecast, num_bins, breaks, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). num_bins number bins PIT histogram. sample-based forecasts, default 10 bins. quantile-based forecasts, default one bin available quantile. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored. 0 1 always added left right bounds, respectively. Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics. integers handle integer forecasts (count data). based methods described Czado et al. (2007). \"nonrandom\" (default) function use non-randomised PIT method. \"random\", use randomised PIT method. \"ignore\", treat integer forecasts continuous. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous integers set random.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"data.table density values bin PIT histogram.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"","code":"library(\"ggplot2\") example <- as_forecast_sample(example_sample_continuous) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit_histogram(example, by = \"model\") ggplot(result, aes(x = mid, y = density)) + geom_col() + facet_wrap(. ~ model) + labs(x = \"Quantile\", \"Density\") # example with quantile data example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit_histogram(example, by = \"model\") ggplot(result, aes(x = mid, y = density)) + geom_col() + facet_wrap(. ~ model) + labs(x = \"Quantile\", \"Density\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-binary-point.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","title":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","text":"Illustration required inputs binary point forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","title":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","text":"Illustration required inputs quantile-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","title":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","text":"Illustration required inputs sample-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.7984288 0.6188294 0.7680146 0.7392137 2.4496739 0.6725337 0.5966822 #> [8] 0.6659581 0.6581834 1.2866256 0.7423596 0.6155816 0.6176548 0.6484069 #> [15] 0.6584252 0.6929587 0.7558694 0.6724925 0.5716195 0.6251716 0.7431658 #> [22] 0.6906088 0.6524924 0.6274762 1.8011311 0.5840689 0.6602319 0.7076748 #> [29] 0.6060337 0.5491802 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.6735755 0.6315584 0.6931931 2.4596254 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 1.1441141 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 1.6335484 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $dispersion #> [1] NA 0.6735755 0.6315584 0.6931931 0.6195326 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 0.5198196 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 0.4719967 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 1.8400928 0.0000000 0.0000000 #> [8] 0.0000000 0.0000000 0.6242945 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 0.0000000 0.0000000 1.1615517 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast_binary","title":"Test whether an object is a forecast object — is_forecast_binary","text":"Test whether object forecast object (see as_forecast() information). can test specific forecast_* class using appropriate is_forecast_* function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"is_forecast_binary(x) is_forecast_nominal(x) is_forecast_point(x) is_forecast_quantile(x) is_forecast_sample(x) is_forecast(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast_binary","text":"x R object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast_binary","text":"is_forecast: TRUE object class forecast, FALSE otherwise. is_forecast_<type>*: TRUE object class forecast_* addition class forecast, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"forecast_binary <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"library(magrittr) # pipe operator log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 example_quantile[observed > 0, ] %>% as_forecast_quantile() %>% transform_forecasts(fun = log_shift, offset = 1) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> <char> <Date> <char> <num> <char> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> <Date> <num> <num> <char> <num> #> 1: <NA> NA NA <NA> NA #> 2: <NA> NA NA <NA> NA #> 3: <NA> NA NA <NA> NA #> 4: <NA> NA NA <NA> NA #> 5: <NA> NA NA <NA> NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> <char> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. log score negative logarithm predictive density evaluated observed value. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: function uses kernel density estimation, well defined integer-valued Monte Carlo Samples. See scoringRules package details alternatives, e.g. calculating scores specific discrete probability distributions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 2.415978 1.391162 2.876486 1.778651 1.990756 3.187023 2.735783 2.308400 #> [9] 2.217484 2.403781 2.688615 2.813455 2.178733 2.080475 2.668287 2.773460 #> [17] 2.619100 2.600304 2.610290 3.177565 3.065729 2.454865 2.860266 2.676266 #> [25] 3.299807 2.918963 2.538826 2.698623 2.858104 2.869216"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 4.4478 5.9304 5.9304 5.9304 5.9304 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments data.table::.data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { # \\dontrun{ df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group( scores, metric, baseline, compare = \"model\", by, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation for counts — pit_histogram_sample","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation for counts — pit_histogram_sample","text":"","code":"pit_histogram_sample( observed, predicted, quantiles, integers = c(\"nonrandom\", \"random\", \"ignore\"), n_replicates = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation for counts — pit_histogram_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. quantiles vector quantiles calculate PIT. integers handle integer forecasts (count data). based methods described Czado et al. (2007). \"nonrandom\" (default) function use non-randomised PIT method. \"random\", use randomised PIT method. \"ignore\", treat integer forecasts continuous. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous integers set random.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation for counts — pit_histogram_sample","text":"vector PIT histogram densities bins corresponding given quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete nonnegative outcomes incidence counts, PIT longer uniform even forecasts ideal. case two methods available ase described Czado et al. (2007). default, nonrandomised PIT calculated using conditional cumulative distribution function $$ F(u) = \\begin{cases} 0 & \\text{} v < P_t(k_t - 1) \\\\ (v - P_t(k_t - 1)) / (P_t(k_t) - P_t(k_t - 1)) & \\text{} P_t(k_t - 1) \\leq v < P_t(k_t) \\\\ 1 & \\text{} v \\geq P_t(k_t) \\end{cases} $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence \\(k\\) time \\(t\\) \\(P_t (-1) = 0\\) definition. Values PIT histogram created averaging \\(n\\) predictions, $$ $$ calculating value bin quantile \\(q_i\\) quantile \\(q_{+ 1}\\) $$ $$ Alternatively, randomised PIT can used instead. case, PIT $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1)) $$ \\(v\\) standard uniform independent \\(k\\). values PIT histogram calculated binning $u_t$ values .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Claudia Czado, Tilmann Gneiting Leonhard Held (2009) Predictive model assessment count data. Biometrika, 96(4), 633-648. Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation for counts — pit_histogram_sample","text":"","code":"## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_histogram_sample(observed, predicted, quantiles = seq(0, 1, 0.1)) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_histogram_sample(observed, predicted, quantiles = seq(0, 1, 0.1)) ## integer predictions, randomised PIT observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_histogram_sample( observed, predicted, quantiles = seq(0, 1, 0.1), integers = \"random\", n_replicates = 30 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations, digits = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations(). digits number indicating many decimal places correlations rounded . default (digits = NULL) rounding takes place.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- scores %>% summarise_scores() %>% get_correlations() plot_correlations(correlations, digits = 2)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) library(magrittr) # pipe operator forecast_counts <- example_quantile %>% as_forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"target_type\", \"target_end_date\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast","title":"Print information about a forecast object — print.forecast","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast","text":"","code":"# S3 method for class 'forecast' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast","text":"","code":"dat <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name #> <char> <Date> <char> <num> <char> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> <Date> <num> <int> <char> <num> #> 1: <NA> NA NA <NA> NA #> 2: <NA> NA NA <NA> NA #> 3: <NA> NA NA <NA> NA #> 4: <NA> NA NA <NA> NA #> 5: <NA> NA NA <NA> NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper/helper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score.forecast_binary","title":"Evaluate forecasts — score.forecast_binary","text":"score() applies selection scoring metrics forecast object (data.table forecasts observations) (see as_forecast()). score() generic dispatches different methods depending class input data. See Forecast types input formats section information forecast types input formats. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_nominal' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_point' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = get_metrics(forecast), ...) score(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score.forecast_binary","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Evaluate forecasts — score.forecast_binary","text":"Customising metrics want pass arguments scoring function, need change scoring function via e.g. purrr::partial() pass updated list functions custom metric metrics argument score(). example, use interval_coverage() interval_range = 90, define new function, e.g. interval_coverage_90 <- purrr::partial(interval_coverage, interval_range = 90) pass new function metrics score(). Note want pass variable argument, can unquote !! make sure value evaluated function created. Consider following example:","code":"custom_arg <- \"foo\" print1 <- purrr::partial(print, x = custom_arg) print2 <- purrr::partial(print, x = !!custom_arg) custom_arg <- \"bar\" print1() # prints 'bar' print2() # prints 'foo'"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Evaluate forecasts — score.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Evaluate forecasts — score.forecast_binary","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score.forecast_binary","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score.forecast_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"library(magrittr) # pipe operator validated <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> <num> <num> <num> <num> <num> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 24101.07031 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 38473.60156 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 27923.81250 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 53.13281 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 233.25781 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 78.47656 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast_quantile( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> <char> <Date> <char> <num> <char> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> <num> <num> <num> <num> <num> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 ae_median #> <lgcl> <lgcl> <num> #> 1: TRUE TRUE 12271 #> 2: FALSE FALSE 25620 #> 3: FALSE TRUE 44192 #> 4: TRUE TRUE 14 #> 5: TRUE TRUE 15 #> --- #> 883: TRUE TRUE 53 #> 884: TRUE TRUE 1 #> 885: FALSE TRUE 46 #> 886: TRUE TRUE 26 #> 887: FALSE TRUE 108 # forecast formats with different metrics if (FALSE) { # \\dontrun{ score(as_forecast_binary(example_binary)) score(as_forecast_quantile(example_quantile)) score(as_forecast_point(example_point)) score(as_forecast_sample(example_sample_discrete)) score(as_forecast_sample(example_sample_continuous)) } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal last/highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 0.169810872 0.319983015 0.720333826 0.136189585 0.539816982 0.573052550 #> [7] 0.210298251 0.005442733 0.940506004 0.209025175 0.132818958 0.775957259 #> [13] 0.533583639 0.773922330 0.177404878 0.032952811 0.700006942 0.860101989 #> [19] 0.010349776 0.035994321 0.423125196 0.650581640 0.408156615 0.150229898 #> [25] 0.003399837 0.300913133 0.070701554 0.002211134 0.008862522 0.099258707 logs_binary(observed, predicted) #> [1] 0.53116635 0.83395161 1.88865474 0.46051080 1.32697839 1.41470349 #> [7] 0.61356527 0.07663796 3.49981031 0.61100092 0.45325406 2.12766051 #> [13] 1.31106851 2.11800405 0.54678893 0.20031742 1.81194692 2.62302230 #> [19] 0.10728887 0.21037750 1.05119662 1.64292444 1.01852104 0.49036148 #> [25] 0.06007715 0.79530283 0.30910680 0.04816419 0.09887158 0.37841454"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Log score for nominal outcomes — logs_nominal","title":"Log score for nominal outcomes — logs_nominal","text":"Log score nominal outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"logs_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log score for nominal outcomes — logs_nominal","text":"observed factor length n N levels holding observed values. predicted nxN matrix predictive probabilities, n (number rows) number observations N (number columns) number possible outcomes. predicted_label factor length N, denoting outcome probabilities predicted correspond .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log score for nominal outcomes — logs_nominal","text":"numeric vector size n log scores","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"factor_levels <- c(\"one\", \"two\", \"three\") predicted_label <- factor(c(\"one\", \"two\", \"three\"), levels = factor_levels) observed <- factor(c(\"one\", \"three\", \"two\"), levels = factor_levels) predicted <- matrix(c(0.8, 0.1, 0.4, 0.1, 0.2, 0.4, 0.1, 0.7, 0.2), nrow = 3) logs_nominal(observed, predicted, predicted_label) #> [1] 0.2231436 0.3566749 0.9162907"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Facilitate evaluation forecasts convenient framework based data.table. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper, doi:10.48550/arXiv.2205.07090 .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor] Michael Chirico michaelchirico4@gmail.com (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 1.120954e+00 5.506171e-01 8.158564e+00 4.953364e-02 1.978748e-01 #> [6] 3.924656e+00 4.315103e-03 1.971522e+00 5.325623e-03 3.438576e+00 #> [11] 4.944164e-02 9.564610e-04 6.898922e-01 3.784738e+00 1.028803e-01 #> [16] 2.632511e-05 3.644782e-01 6.029127e-01 8.747270e-01 2.939886e-01 #> [21] 2.574917e-04 5.380933e+00 1.563555e+00 1.536483e+00 1.565979e-02 #> [26] 1.691586e-01 1.467380e+01 2.184548e+00 1.223069e+00 1.486889e-01"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = get_metrics(example_binary), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x55dc54ba2740> #> <environment: namespace:scoringutils> #> select_metrics( metrics = get_metrics(example_binary), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x55dc54ba2740> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called constructing forecast object (see as_forecast()) via forecast_unit argument . done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"library(magrittr) # pipe operator example_quantile %>% scoringutils:::set_forecast_unit( c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, horizon, and model #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted location target_end_date target_type #> <num> <num> <int> <char> <Date> <char> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> <num> <char> #> 1: NA <NA> #> 2: NA <NA> #> 3: NA <NA> #> 4: NA <NA> #> 5: NA <NA> #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", fun = mean, ...) summarize_scores(scores, by = \"model\", fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default \"model\", .e. scores summarised \"model\" column. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"library(magrittr) # pipe operator scores <- example_sample_continuous %>% as_forecast_sample() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores, by = \"model\") #> model bias dss crps overprediction #> <char> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 5281.81845 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 6701.83844 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 6986.16572 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 11.20505 #> underprediction dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> <num> #> 1: 2433.20525 2161.93515 10.747811 8763.6176 12406.03100 2.103026e+09 #> 2: 5864.53649 2743.31134 Inf 9680.3792 18932.50196 2.885063e+09 #> 3: 1740.93388 3174.33393 Inf 12999.5404 14680.12285 3.152268e+09 #> 4: 19.28195 29.70318 5.941622 123.6211 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 #> overprediction underprediction dispersion log_score mad ae_median #> <num> <num> <num> <num> <num> <num> #> 1: 10552.97603 4861.015121 4289.06407 15.633420 17385.2629 24749.39707 #> 2: 13346.03509 11727.330575 5380.21780 Inf 18982.2128 37648.01693 #> 3: 13462.90822 3346.583024 6087.02483 Inf 24929.3438 28233.04536 #> 4: 10.66088 5.395380 34.80623 5.862203 141.9723 62.66492 #> 5: 57.64179 1.742401 106.40489 6.977391 378.5457 216.98699 #> 6: 11.20505 19.281946 29.70318 5.941622 123.6211 79.66001 #> 7: 19.58555 13.849095 41.35549 6.024092 167.4829 102.18939 #> se_mean #> <num> #> 1: 4.206042e+09 #> 2: 5.769964e+09 #> 3: 6.082863e+09 #> 4: 1.080233e+04 #> 5: 1.622417e+05 #> 6: 1.371418e+04 #> 7: 3.243111e+04 # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps overprediction #> <char> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 37275.23950 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 39070.74445 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 40776.81690 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 21.34675 #> underprediction dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> <num> #> 1: 8634.87723 5163.42293 21.510119 19799.1620 42801.64123 1.564286e+10 #> 2: 20537.03929 3664.87255 NaN 13610.4174 49458.36446 1.760651e+10 #> 3: 8096.39644 7266.11787 NaN 29616.1714 51129.54601 2.209086e+10 #> 4: 36.98584 29.60927 1.126019 123.3465 76.09471 2.994664e+04 # round digits summarise_scores(scores, by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps overprediction underprediction #> <char> <num> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 5300 2400 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 6700 5900 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 7000 1700 #> 4: UMass-MechBayes -0.0270 10 60 11 19 #> dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> #> 1: 2200 11.0 8800 12000 2.1e+09 #> 2: 2700 Inf 9700 19000 2.9e+09 #> 3: 3200 Inf 13000 15000 3.2e+09 #> 4: 30 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754302 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419360 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950677 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718827 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609898 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677099 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast_quantile() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754302 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419360 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950677 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718827 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609898 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677099 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754310 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419580 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950683 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718991 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609907 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677269 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> <num> <num> <num> <num> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> ae_median #> <num> #> 1: 8880.542969 #> 2: 16156.871094 #> 3: 11208.072874 #> 4: 78.476562 #> 5: 22.458900 #> 6: 39.185406 #> 7: 25.585018 #> 8: 2.069103 # adding multiple transformations example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 127300 Germany <NA> #> 2: DE 2021-01-02 Deaths 4534 Germany <NA> #> 3: DE 2021-01-09 Cases 154922 Germany <NA> #> 4: DE 2021-01-09 Deaths 6117 Germany <NA> #> 5: DE 2021-01-16 Cases 110183 Germany <NA> #> 6: DE 2021-01-16 Deaths 5867 Germany <NA> #> quantile_level predicted model horizon scale #> <num> <num> <char> <num> <char> #> 1: NA NA <NA> NA natural #> 2: NA NA <NA> NA natural #> 3: NA NA <NA> NA natural #> 4: NA NA <NA> NA natural #> 5: NA NA <NA> NA natural #> 6: NA NA <NA> NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). set \\(k = 1, \\dots, K\\) prediction intervals median \\(m\\), can compute weighted interval score (WIS) sum interval scores individual intervals: $$ \\text{WIS}_{\\alpha_{\\{0:K\\}}}(F, y) = \\frac{1}{K + 1/2} \\times \\left(w_0 \\times |y - m| + \\sum_{k=1}^{K} \\left\\{ w_k \\times \\text{}_{\\alpha_k}(F, y) \\right\\}\\right) $$ individual scores usually weighted \\(w_k = \\frac{\\alpha_k}{2}\\). weight ensures increasing number equally spaced quantiles, WIS converges continuous ranked probability score (CRPS). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE ) dispersion_quantile(observed, predicted, quantile_level, ...) overprediction_quantile(observed, predicted, quantile_level, ...) underprediction_quantile(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction_quantile(), underprediction_quantile() dispersion_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion_quantile(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation)","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Weighted interval score (WIS) — wis","text":"Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new set as_forecast_<type>() functions introduced validate data convert inputs forecast object (data.table forecast class additional class corresponding forecast type (see )). Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile, forecast_sample, forecast_nominal, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using functions as_forecast_point(), as_forecast_binary(), as_forecast_quantile(), as_forecast_sample() (replace previous check_forecast()). forecast object data.table class forecast additional class corresponding forecast type (e.g. forecast_quantile). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. binary point forecasts, columns “observed” “predicted” now removed consistency forecast types. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using function get_metrics(), generic S3 methods forecast type. returns named list scoring rules suitable respective forecast object. example, call get_metrics(example_quantile). Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using purrr::partial() select_metric(). See ?score() information. CRPS now reported decomposition dispersion, overprediction underprediction. functionality calculate Probability Integral Transform (PIT) deprecated replaced functionality calculate PIT histograms, using get_pit_histogram() function; part change, nonrandomised PITs can now calculated count data, done default","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"as_forecast_<type>() functions create forecast object validates . also allow users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases. See ?as_forecast() information. as_forecast_<type>() functions like e.g. as_forecast_point() as_forecast_quantile() S3 methods converting another forecast type respective forecast type. example, as_forecast_quantile() method converting forecast_sample object forecast_quantile object estimating quantiles samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast_quantile( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast_<type>(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, users can call `as_forecast_<type>() re-validate forecast object. Simply printing object also provide additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast_quantile() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts(). Renamed interval_coverage_quantile() interval_coverage(). “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se.get_duplicate_forecasts() now sorts outputs according forecast unit, making easier spot duplicates. addition, counts option allows user display number duplicates forecast unit, rather raw duplicated rows. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring. Function set_forecast_unit() deleted. Instead now forecast_unit argument as_forecast_<type>() well get_duplicate_forecasts(). Removed interval_coverage_dev_quantile(). Users can still access difference nominal actual interval coverage using get_coverage(). pit(), pit_sample() plot_pit() removed replaced functionality create PIT histograms (pit_histogram_sampel() get_pit_histogram())","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles. contrast previous correlation function, get_correlations doesn’t round correlations default. Instead, plot_correlations now digits argument allows users round correlations plotting . Alternatively, using dplyr, call something like mutate(correlations, across((.numeric), \\(x) signif(x, digits = 2))) output get_correlations. wis() now errors default quantile levels form valid prediction intervals returns NA missing values. Previously, na.rm set TRUE default, lead unexpected results, users aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs. scoringutils now requires R >= 4.0","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug as_forecast_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}] +[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @returns a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) library(ggdist) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (\"quantile_level\" %in% colnames(data)) { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (\"sample_id\" %in% colnames(data)) { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast_sample(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast_quantile(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast_quantile() %>% score(metrics = list(wis = wis, dispersion = dispersion_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @returns A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast_quantile(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast_quantile(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form S(ŷ,y)S(\\hat{y}, y), ŷ\\hat{y} forecast yy observation. probabilistic forecasts, usually take form S(F,y)S(F, y), FF cumulative density function (CDF) predictive distribution yy observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let GG true, unobserved data-generating distribution. scoring rule said proper, GG ideal forecast F=GF = G, forecast F′≠FF' \\neq F expectation receives better score FF. scoring rule considered strictly proper , GG, forecast F′F' expectation receives score better FF.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling get_metrics(example_point). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. ae=|y−ŷ|\\text{ae} = |y - \\hat{y}| absolute error appropriate rule ŷ\\hat{y} corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. se=(y−ŷ)2\\text{se} = (y - \\hat{y})^2 squared error appropriate rule ŷ\\hat{y} corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. ape=|y−ŷ||y|\\text{ape} = \\frac{|y - \\hat{y}|}{|y|} absolute percentage error appropriate rule ŷ\\hat{y} corresponds β\\beta-median forecaster’s predictive distribution β=−1\\beta = -1. β\\beta-median, med(β)(F)\\text{med}^{(\\beta)}(F), median random variable whose density proportional yβf(y)y^\\beta f(y). specific β\\beta-median corresponds absolute percentage error med(−1)(F)\\text{med}^{(-1)}(F). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?get_metrics(example_binary). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. BS(p,y)=(p−y)2={p2,y=1(1−p)2,y=0\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation} Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. Log score(p,y)=−log(1−|y−p|)={−log(p),y=1−log(1−p),y=0\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation} log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling get_metrics(example_sample_continuous). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined CRPS(F,y)=∫−∞∞(F(x)−1(x≥y))2dx,\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx, yy observed value FF CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : RPS(P,y)=∑x=0∞(P(x)−1(x≥y))2, \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, PP cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Sample-based forecasts > CRPS","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"CRPS can interpreted sum dispersion, overprediction underprediction component. mm median forecast dispersion component CRPS(F,m),\\text{CRPS}(F, m), overprediction component {m>yCRPS(F,y)−CRPS(F,m)m≤y0 \\begin{cases} m > y & CRPS(F, y) - CRPS(F, m)\\\\ m \\leq y & 0\\\\ \\end{cases} underprediction component {m<yCRPS(F,y)−CRPS(F,m)m≥y0 \\begin{cases} m < y & CRPS(F, y) - CRPS(F, m)\\\\ m \\geq y & 0\\\\ \\end{cases} can accessed via dispersion_sample(), overprediction_sample() underprediction_sample() functions, respectively.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"log-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Log score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. logarithmic scoring rule simply negative logarithm density predictive distribution evaluated observed value: log score(F,y)=−logf(y), \\text{log score}(F, y) = -\\log f(y), ff predictive probability density function (PDF) corresponding Forecast FF yy observed value. discrete forecasts, log score can computed log score(F,y)=−logpy, \\text{log score}(F, y) = -\\log p_y, pyp_y probability assigned observed outcome yy forecast FF. logarithmic scoring rule can produce large penalties observed value takes values f(y)f(y) (pyp_y) close zero. therefore considered sensitive outlier forecasts. may desirable applications, also means scores can easily dominated extreme values. logarithmic scoring rule local scoring rule, meaning score depends probability assigned actual outcome. often regarded desirable property example context Bayesian inference . implies example, ranking forecasters invariant monotone transformations predictive distribution target. scoringutils re-exports logs_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. One implications currently advisable use log score discrete forecasts. reason scoringRules::logs_sample() estimates predictive density samples, can problematic discrete forecasts. See ?logs_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dawid-sebastiani-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dawid-Sebastiani score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: FF. predictive distribution mean μ\\mu standard deviation σ\\sigma. Dawid-Sebastiani score proper scoring rule relies first moments predictive distribution therefore easy compute. given dss(F,y)=(y−μσ)2+2⋅logσ.\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma. scoringutils re-exports implementation DSS scoringRules package. assumes forecast represented set samples drawn predictive distribution. See ?dss_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dispersion---median-absolute-deviation-mad","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dispersion - Median Absolute Deviation (MAD)","title":"Scoring rules in `scoringutils`","text":"Observation: required. Forecast: FF, predictive distribution. Dispersion (also called sharpness) ability produce narrow forecasts. feature forecasts depend observations. Dispersion therefore interest conditional calibration: precise forecast useful clearly wrong. One way measure sharpness (suggested Funk et al. (2019)) normalised median absolute deviation median (MADN) ). computed S(F)=10.675⋅median(|F−median(F)|). S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|F - \\text{median(F)}|). forecast FF follows normal distribution, sharpness equal standard deviation FF.details, see ?mad_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. Bias measure tendency forecaster - underpredict. continuous forecasts, scoringutils implementation calculates bias B(F,y)=1−2⋅F(y),B(F, y) = 1 - 2 \\cdot F (y), F(y)F(y) cumulative distribution function forecast. discrete forecasts, calculate bias B(P,y)=1−(P(y)+P(y+1)).B(P, y) = 1 - (P(y) + P(y + 1)). P(y)P(y) cumulative probability assigned outcomes smaller equal yy, .e. cumulative probability mass function. Bias bound -1 1 represents tendency forecasts biased rather absolute amount - underprediction (e.g. case weighted interval score (WIS, see ).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF aemedian(F,y)=|median(F)−y|.\\text{ae}_{\\text{median}}(F, y) = |\\text{median} (F) - y|. See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error-of-the-mean","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Squared error of the mean","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF semedn(F,y)=(mean(F)−y)2.\\text{se}_{\\text{medn}}(F, y) = (\\text{mean} (F) - y)^2. See section note caution Gneiting (2011) discussion correspondence squared error mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Quantile-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics quantile-based forecasts calling get_metrics(example_quantile). overview input output formats quantile forecasts: Input output formats: metrics quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"weighted-interval-score-wis","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Weighted interval score (WIS)","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form lower (ll) upper (uu) bounds central prediction intervals. weighted interval score (WIS) strictly proper scoring rule can understood approximation CRPS forecasts quantile format (turn represents generalisation absolute error). Quantiles assumed lower upper bounds prediction intervals symmetric around median. single interval, interval score ISα(F,y)=(u−l)⏟dispersion+2α⋅(l−y)⋅𝟏(y≤l)⏟overprediction+2α⋅(y−u)⋅𝟏(y≥u)⏟underprediction,IS_\\alpha(F,y) = \\underbrace{(u-l)}_\\text{dispersion} + \\underbrace{\\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l)}_{\\text{overprediction}} + \\underbrace{\\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u)}_{\\text{underprediction}}, 𝟏()\\mathbf{1}() indicator function, ll uu α2\\frac{\\alpha}{2} 1−α21 - \\frac{\\alpha}{2} quantiles predictive distribution FF. ll uu together form prediction interval. interval score can understood sum three components: dispersion, overprediction underprediction. set KK prediction intervals median mm, score given weighted sum individual interval scores, .e. WIS=1K+0.5⋅(w0⋅|y−m|+∑k=1Kwk⋅ISαk(F,y)),WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha_{k}}(F, y)\\right), mm median forecast wkw_k weight assigned every interval. weights set wk=αk2w_k = \\frac{\\alpha_k}{2} w0=0.5w_0 = 0.5, WIS converges CRPS increasing number equally spaced quantiles. See ?wis() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion-1","dir":"Articles","previous_headings":"Quantile-based forecasts > Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction_quantile(), ?underprediction_quantile() ?dispersion_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles, QQ. Bias can measured B(F,y)={(1−2⋅max{α|qα∈Q∧qα≤y}),y<q0.5(overprediction)(1−2⋅min{α|qα∈Qt∧qα≥y},y>q0.5(underprediction)0,y=q0.5,\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation} qαq_\\alpha α\\alpha-quantile predictive distribution. consistency, define QQ (set quantiles form predictive distribution FF) always includes element q0=−∞q_0 = -\\infty q1=∞q_1 = \\infty. clearer terms, bias : 1−(2×1 - (2 \\times maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. 1−(2×1 - (2 \\times minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. 00if observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation 0.6=−0.5=0.1.0.6 = - 0.5 = 0.1. interval coverage deviation=observed interval coverage−nominal interval coverage\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. aemedian=|median(F)−y|\\text{ae}_\\text{median} = |\\text{median}(F) - y| See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level τ\\tau defined QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,y≤qτ2⋅τ*|qτ−y|,y>qτ,\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation} qτq_\\tau τ\\tau-quantile predictive distribution FF, 𝟏(⋅)\\mathbf{1}(\\cdot) indicator function. (unweighted) interval score (see ) 1−α1 - \\alpha prediction interval can computed quantile scores levels α/2\\alpha/2 1−α/21 - \\alpha/2 ISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)α\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}. weighted interval score can obtained simple average quantile scores: WISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)2\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author. Michael Chirico. Contributor.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast_quantile( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ) ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted location forecast_date target_end_date #> <num> <num> <int> <char> <Date> <Date> #> 1: 127300 NA NA DE <NA> 2021-01-02 #> 2: 4534 NA NA DE <NA> 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> <char> <char> <num> #> 1: Cases <NA> NA #> 2: Deaths <NA> NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast_<type>() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> <num> <num> <num> <num> <num> #> 1: 3660.0 -0.05640 0.391 0.805 24100.0 #> 2: 4100.0 0.09800 0.328 0.820 38500.0 #> 3: 5660.0 -0.07890 0.469 0.789 27900.0 #> 4: 30.2 0.07270 0.875 1.000 53.1 #> 5: 91.4 0.33900 0.664 1.000 233.0 #> 6: 26.9 -0.02230 0.461 0.875 78.5 #> 7: 31.9 -0.00513 0.420 0.908 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(<citation>, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"funding","dir":"","previous_headings":"","what":"Funding","title":"Utilities for Scoring and Assessing Predictions","text":"development scoringutils funded via Health Protection Research Unit (grant code NIHR200908) Wellcome Trust (grant: 210758/Z/18/Z). work also supported US National Institutes General Medical Sciences (R35GM119582). content solely responsibility authors necessarily represent official views NIGMS, National Institutes Health.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributors","dir":"","previous_headings":"","what":"Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"contributions project gratefully acknowledged using allcontributors package following -contributors specification. Contributions kind welcome!","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code","dir":"","previous_headings":"Contributors","what":"Code","title":"Utilities for Scoring and Assessing Predictions","text":"nikosbosse, seabbs, jamesmbaazam, Bisaloo, sbfnk, actions-user, toshiakiasakura, MichaelChirico, nickreich, jhellewell14, damonbayer","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-authors","dir":"","previous_headings":"Contributors","what":"Issue Authors","title":"Utilities for Scoring and Assessing Predictions","text":"DavideMagno, mbojan, dshemetov, elray1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-contributors","dir":"","previous_headings":"Contributors","what":"Issue Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"jbracher, dylanhmorris, jonathonmellor, kathsherratt","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ |observed - median prediction| $$ median prediction predicted value quantile_level == 0.5. function requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$ |observed - median\\_prediction| $$ median prediction calculated median predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Numeric vector length n absolute errors median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using purrr::partial().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for binary forecasts — as_forecast_binary","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. arguments observed, predicted, etc. make possible rename existing columns input data match required columns forecast object. Using argument forecast_unit, can specify columns uniquely identify single forecast (thereby removing , unneeded columns. See section \"Forecast Unit\" details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"","code":"as_forecast_binary( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"forecast object class forecast_binary","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"required-input","dir":"Reference","previous_headings":"","what":"Required input","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"input needs data.frame similar following columns: observed: factor exactly two levels representing observed values. highest factor level assumed reference level. means corresponding value predicted represent probability observed value equal highest factor level. predicted: numeric predicted probabilities, representing probability corresponding value observed equal highest available factor level. convenience, recommend additional column model holding name forecaster model produced prediction, strictly necessary. See example_binary data set example.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"","code":"as_forecast_binary( example_binary, predicted = \"predicted\", forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> predicted observed model target_type target_end_date #> <num> <fctr> <char> <char> <Date> #> 1: NA <NA> <NA> Cases 2021-01-02 #> 2: NA <NA> <NA> Deaths 2021-01-02 #> 3: NA <NA> <NA> Cases 2021-01-09 #> 4: NA <NA> <NA> Deaths 2021-01-09 #> 5: NA <NA> <NA> Cases 2021-01-16 #> --- #> 1027: 0.250 0 EuroCOVIDhub-baseline Deaths 2021-07-24 #> 1028: 0.475 0 UMass-MechBayes Deaths 2021-07-24 #> 1029: 0.450 0 UMass-MechBayes Deaths 2021-07-24 #> 1030: 0.375 0 epiforecasts-EpiNow2 Deaths 2021-07-24 #> 1031: 0.300 0 epiforecasts-EpiNow2 Deaths 2021-07-24 #> horizon location #> <num> <char> #> 1: NA DE #> 2: NA DE #> 3: NA DE #> 4: NA DE #> 5: NA DE #> --- #> 1027: 2 IT #> 1028: 3 IT #> 1029: 2 IT #> 1030: 3 IT #> 1031: 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_doc_template.html","id":null,"dir":"Reference","previous_headings":"","what":"General information on creating a forecast object — as_forecast_doc_template","title":"General information on creating a forecast object — as_forecast_doc_template","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. arguments observed, predicted, etc. make possible rename existing columns input data match required columns forecast object. Using argument forecast_unit, can specify columns uniquely identify single forecast (thereby removing , unneeded columns. See section \"Forecast Unit\" details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_doc_template.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General information on creating a forecast object — as_forecast_doc_template","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_doc_template.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"General information on creating a forecast object — as_forecast_doc_template","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Common functionality for as_forecast_<type> functions — as_forecast_generic","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"Common functionality as_forecast_<type> functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"","code":"as_forecast_generic( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Common functionality for as_forecast_<type> functions — as_forecast_generic","text":"function splits part functionality as_forecast_<type> as_forecast_<type> functions. renames required columns, appropriate, sets forecast unit.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for nominal forecasts — as_forecast_nominal","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. arguments observed, predicted, etc. make possible rename existing columns input data match required columns forecast object. Using argument forecast_unit, can specify columns uniquely identify single forecast (thereby removing , unneeded columns. See section \"Forecast Unit\" details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"","code":"as_forecast_nominal( data, forecast_unit = NULL, observed = NULL, predicted = NULL, predicted_label = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". predicted_label (optional) Name column data denotes outcome predicted probability corresponds . column renamed \"predicted_label\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"forecast object class forecast_nominal","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"Nominal forecasts form categorical forecasts represent generalisation binary forecasts multiple outcomes. possible outcomes observed values can assume ordered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"required-input","dir":"Reference","previous_headings":"","what":"Required input","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"input needs data.frame similar following columns: observed: Column observed values type factor N levels, N number possible outcomes. levels factor represent possible outcomes observed values can assume. predicted: numeric column predicted probabilities. values represent probability observed value equal factor level denoted predicted_label. Note forecasts must complete, .e. must probability assigned every possible outcome probabilities must sum one. predicted_label: factor N levels, denoting outcome probabilities predicted correspond . convenience, recommend additional column model holding name forecaster model produced prediction, strictly necessary. See example_nominal data set example.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"","code":"as_forecast_nominal( na.omit(example_nominal), predicted = \"predicted\", forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> Forecast type: nominal #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Warning: ! Error in validating forecast object: Error in assert_forecast(forecast = out, #> verbose = FALSE) : ! Found incomplete forecasts ℹ For a nominal forecast, all #> possible outcomes must be assigned a probability explicitly. ℹ Found first #> missing probabilities in the forecast identified by model == NA, target_type #> == NA, target_end_date == NA, horizon == NA, and location == NA #> observed predicted_label predicted model target_type #> <fctr> <fctr> <num> <char> <char> #> 1: low low 0.525 EuroCOVIDhub-ensemble Cases #> 2: low low 0.075 EuroCOVIDhub-baseline Cases #> 3: low low 0.150 epiforecasts-EpiNow2 Cases #> 4: medium low 0.100 EuroCOVIDhub-ensemble Deaths #> 5: medium low 0.275 EuroCOVIDhub-baseline Deaths #> --- #> 2657: low medium 0.300 EuroCOVIDhub-baseline Deaths #> 2658: medium medium 0.850 UMass-MechBayes Deaths #> 2659: low medium 0.825 UMass-MechBayes Deaths #> 2660: medium medium 0.275 epiforecasts-EpiNow2 Deaths #> 2661: low medium 0.375 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> <Date> <num> <char> #> 1: 2021-05-08 1 DE #> 2: 2021-05-08 1 DE #> 3: 2021-05-08 1 DE #> 4: 2021-05-08 1 DE #> 5: 2021-05-08 1 DE #> --- #> 2657: 2021-07-24 2 IT #> 2658: 2021-07-24 3 IT #> 2659: 2021-07-24 2 IT #> 2660: 2021-07-24 3 IT #> 2661: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for point forecasts — as_forecast_point","title":"Create a forecast object for point forecasts — as_forecast_point","text":"converting forecast_quantile object forecast_point object, 0.5 quantile extracted returned point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for point forecasts — as_forecast_point","text":"","code":"as_forecast_point(data, ...) # Default S3 method as_forecast_point( data, forecast_unit = NULL, observed = NULL, predicted = NULL, ... ) # S3 method for class 'forecast_quantile' as_forecast_point(data, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for point forecasts — as_forecast_point","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object for point forecasts — as_forecast_point","text":"forecast object class forecast_point","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"required-input","dir":"Reference","previous_headings":"","what":"Required input","title":"Create a forecast object for point forecasts — as_forecast_point","text":"input needs data.frame similar following columns: observed: Column type numeric observed values. predicted: Column type numeric predicted values. convenience, recommend additional column model holding name forecaster model produced prediction, strictly necessary. See example_point data set example.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. arguments observed, predicted, etc. make possible rename existing columns input data match required columns forecast object. Using argument forecast_unit, can specify columns uniquely identify single forecast (thereby removing , unneeded columns. See section \"Forecast Unit\" details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"","code":"as_forecast_quantile(data, ...) # Default S3 method as_forecast_quantile( data, forecast_unit = NULL, observed = NULL, predicted = NULL, quantile_level = NULL, ... ) # S3 method for class 'forecast_sample' as_forecast_quantile( data, probs = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. probs numeric vector quantile levels quantiles computed. Corresponds probs argument quantile(). type Type argument passed quantile function. information, see quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"forecast object class forecast_quantile","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"required-input","dir":"Reference","previous_headings":"","what":"Required input","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"input needs data.frame similar following columns: observed: Column type numeric observed values. predicted: Column type numeric predicted values. Predicted values represent quantiles predictive distribution. quantile_level: Column type numeric, denoting quantile level corresponding predicted value. Quantile levels must 0 1. convenience, recommend additional column model holding name forecaster model produced prediction, strictly necessary. See example_quantile data set example.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"converting-from-forecast-sample-to-forecast-quantile","dir":"Reference","previous_headings":"","what":"Converting from forecast_sample to forecast_quantile","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"creating forecast_quantile object forecast_sample object, quantiles estimated computing empircal quantiles samples via quantile(). Note empirical quantiles biased estimator true quantiles particular tails distribution number available samples low.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"","code":"as_forecast_quantile( example_quantile, predicted = \"predicted\", forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted model target_type #> <num> <num> <int> <char> <char> #> 1: 127300 NA NA <NA> Cases #> 2: 4534 NA NA <NA> Deaths #> 3: 154922 NA NA <NA> Cases #> 4: 6117 NA NA <NA> Deaths #> 5: 110183 NA NA <NA> Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> <Date> <num> <char> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for sample-based forecasts — as_forecast_sample","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. arguments observed, predicted, etc. make possible rename existing columns input data match required columns forecast object. Using argument forecast_unit, can specify columns uniquely identify single forecast (thereby removing , unneeded columns. See section \"Forecast Unit\" details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"","code":"as_forecast_sample( data, forecast_unit = NULL, observed = NULL, predicted = NULL, sample_id = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". sample_id (optional) Name column data contains sample id. column renamed \"sample_id\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"forecast object class forecast_sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"required-input","dir":"Reference","previous_headings":"","what":"Required input","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"input needs data.frame similar following columns: observed: Column type numeric observed values. predicted: Column type numeric predicted values. Predicted values represent random samples predictive distribution. sample_id: Column type unique identifiers (unique within single forecast) sample. convenience, recommend additional column model holding name forecaster model produced prediction, strictly necessary. See example_sample_continuous example_sample_discrete data set example","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Assert object forecast object (.e. data.table class forecast additional class forecast_<type> corresponding forecast type). See corresponding assert_forecast_<type> functions details required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values). forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Returns NULL invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"forecast <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object. actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Function assesses whether inputs correspond requirements scoring nominal forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"","code":"assert_input_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"observed Input checked. factor length n N levels holding observed values. n number observations N number possible outcomes observed values can assume. output) predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level. predicted_label Factor length N N levels, N number possible outcomes observed values can assume.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — assert_scores","title":"Validate an object of class scores — assert_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — assert_scores","text":"","code":"assert_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — assert_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — assert_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed linear interpolation two innermost quantiles. median available imputed, error thrown. Note order compute bias, quantiles must non-decreasing increasing quantile levels. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame (similar) predicted observed values. See details section additional information required input format.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of rows — check_number_per_forecast","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Helper function checks number rows (corresponding e.g quantiles samples) per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values). copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare a subset of common forecasts — compare_forecasts","title":"Compare a subset of common forecasts — compare_forecasts","text":"function compares two comparators based subset forecasts comparators made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple comparators single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare a subset of common forecasts — compare_forecasts","text":"","code":"compare_forecasts( scores, compare = \"model\", name_comparator1, name_comparator2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare a subset of common forecasts — compare_forecasts","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. name_comparator1 Character, name first comparator name_comparator2 Character, name comparator compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare a subset of common forecasts — compare_forecasts","text":"list mean score ratios p-values comparison two comparators","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare a subset of common forecasts — compare_forecasts","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts Continuous ranked probability score (CRPS) can interpreted sum three components: overprediction, underprediction dispersion. \"Dispersion\" defined CRPS median forecast $m$. observation $y$ greater $m$ overpredictoin defined CRPS forecast $y$ minus dispersion component, underprediction zero. , hand, $y<m$ underprediction defined CRPS forecast $y$ minus dispersion component, overprediction zero. overprediction, underprediction dispersion components correspond wis().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"crps_sample(observed, predicted, separate_results = FALSE, ...) dispersion_sample(observed, predicted, ...) overprediction_sample(observed, predicted, ...) underprediction_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"(Continuous) ranked probability score — crps_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. separate_results Logical. TRUE (default FALSE), separate parts CRPS (dispersion penalty, penalties - -prediction) get returned separate elements list. want data.frame instead, simply call .data.frame() output. ... Additional arguments passed crps_sample() functions overprediction_sample(), underprediction_sample() dispersion_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"(Continuous) ranked probability score — crps_sample","text":"Vector scores. dispersion_sample(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Continuous) ranked probability score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(observed, predicted) #> [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"object class forecast_binary (see as_forecast_binary()) following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Nominal example data — example_nominal","title":"Nominal example data — example_nominal","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Nominal example data — example_nominal","text":"","code":"example_nominal"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Nominal example data — example_nominal","text":"object class forecast_nominal (see as_forecast_nominal()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made predicted_label outcome probabilty corresponds predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Nominal example data — example_nominal","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Nominal example data — example_nominal","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"object class forecast_point (see as_forecast_point()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"object class forecast_quantile (see as_forecast_quantile()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"object class forecast_sample (see as_forecast_sample()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"object class forecast_sample (see as_forecast_sample()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe_archive/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics.scores(scores), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores) #> wis overprediction underprediction dispersion bias #> <num> <num> <num> <num> <num> #> 1: 1.0000000 0.94297565 0.28377361 0.45566303 0.10545891 #> 2: 0.9429757 1.00000000 -0.03310356 0.32493799 0.21532161 #> 3: 0.2837736 -0.03310356 1.00000000 0.14580143 -0.35123801 #> 4: 0.4556630 0.32493799 0.14580143 1.00000000 0.11118365 #> 5: 0.1054589 0.21532161 -0.35123801 0.11118365 1.00000000 #> 6: -0.2076649 -0.14556039 -0.21392764 -0.09400664 0.01338140 #> 7: -0.4075613 -0.31824017 -0.35756699 -0.08614678 0.09802725 #> 8: 0.9886108 0.90326672 0.33589892 0.53809741 0.09578751 #> interval_coverage_50 interval_coverage_90 ae_median metric #> <num> <num> <num> <char> #> 1: -0.20766492 -0.40756133 0.98861080 wis #> 2: -0.14556039 -0.31824017 0.90326672 overprediction #> 3: -0.21392764 -0.35756699 0.33589892 underprediction #> 4: -0.09400664 -0.08614678 0.53809741 dispersion #> 5: 0.01338140 0.09802725 0.09578751 bias #> 6: 1.00000000 0.37245118 -0.24559356 interval_coverage_50 #> 7: 0.37245118 1.00000000 -0.41079097 interval_coverage_90 #> 8: -0.24559356 -0.41079097 1.00000000 ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast_quantile() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values). character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> <char> <num> <num> <num> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> <num> <num> <num> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Internal helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data, forecast_unit = NULL, counts = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. counts output show number duplicates per forecast unit instead individual duplicated rows? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 5: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 6: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 13: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 14: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> <num> <int> <char> <num> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.950 1464 epiforecasts-EpiNow2 1 #> 3: 0.975 1642 epiforecasts-EpiNow2 1 #> 4: 0.975 1642 epiforecasts-EpiNow2 1 #> 5: 0.990 1951 epiforecasts-EpiNow2 1 #> 6: 0.990 1951 epiforecasts-EpiNow2 1 #> 7: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 8: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 9: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 10: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 11: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 12: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 13: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 14: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 15: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 16: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 17: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 18: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 19: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 20: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 21: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values). character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_forecast_counts(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: <model, target_type> #> model target_type count #> <char> <char> <int> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get forecast type from forecast object — get_forecast_type","title":"Get forecast type from forecast object — get_forecast_type","text":"Get forecast type forecast object","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get forecast type from forecast object — get_forecast_type","text":"","code":"get_forecast_type(forecast)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get forecast type from forecast object — get_forecast_type","text":"forecast forecast object (validated data.table predicted observed values).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get forecast type from forecast object — get_forecast_type","text":"Character vector length one forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section additional information required input format.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, using forecast_unit argument. simply drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for binary forecasts — get_metrics.forecast_binary","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"binary forecasts, default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast_binary()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"list scoring functions.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"get_metrics(example_binary) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x5626cc441010> #> <environment: namespace:scoringutils> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> <bytecode: 0x5626cc4401d8> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x5626cc441010> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x5626cc441010> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"nominal forecasts, default scoring rule : \"log_score\" = logs_nominal()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"# S3 method for class 'forecast_nominal' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"x forecast object (validated data.table predicted observed values, see as_forecast_binary()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"get_metrics(example_nominal) #> $log_score #> function (observed, predicted, predicted_label) #> { #> assert_input_nominal(observed, predicted, predicted_label) #> n <- length(observed) #> if (n == 1) { #> predicted <- matrix(predicted, nrow = 1) #> } #> observed_indices <- as.numeric(observed) #> pred_for_observed <- predicted[cbind(1:n, observed_indices)] #> logs <- -log(pred_for_observed) #> return(logs) #> } #> <bytecode: 0x5626c9ce0070> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for point forecasts — get_metrics.forecast_point","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"point forecasts, default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape() note caution: Every scoring rule point forecast implicitly minimised specific aspect predictive distribution (see Gneiting, 2011). mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, results misleading. Failure respect correspondence can lead grossly misleading results! Consider example section .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"# S3 method for class 'forecast_point' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"x forecast object (validated data.table predicted observed values, see as_forecast_binary()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"Making Evaluating Point Forecasts, Gneiting, Tilmann, 2011, Journal American Statistical Association.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"get_metrics(example_point, select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> <bytecode: 0x5626cdf8d760> #> <environment: namespace:Metrics> #> library(magrittr) set.seed(123) n <- 500 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) df <- data.frame( model = rep(c(\"perfect\", \"bad\"), each = n), predicted = c(rep(predicted_mu, n), predicted_not_mu), observed = rep(observed, 2), id = rep(1:n, 2) ) %>% as_forecast_point() score(df) %>% summarise_scores() #> model ae_point se_point ape #> <char> <num> <num> <num> #> 1: perfect 34.64686 2145.813 3543.184 #> 2: bad 32.34199 2238.566 2692.868"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"quantile-based forecasts, default scoring rules : \"wis\" = wis() \"overprediction\" = overprediction_quantile() \"underprediction\" = underprediction_quantile() \"dispersion\" = dispersion_quantile() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = purrr::partial( interval_coverage, interval_range = 90 ) \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function purrr::partial(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"x forecast object (validated data.table predicted observed values, see as_forecast_binary()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"get_metrics(example_quantile, select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> interval_ranges <- get_range_from_quantile(quantile_level[quantile_level != #> 0.5]) #> complete_intervals <- duplicated(interval_ranges) | duplicated(interval_ranges, #> fromLast = TRUE) #> if (!all(complete_intervals) && !isTRUE(na.rm)) { #> incomplete <- quantile_level[quantile_level != 0.5][!complete_intervals] #> cli_abort(c(`!` = \"Not all quantile levels specified form symmetric prediction\\n intervals.\\n The following quantile levels miss a corresponding lower/upper bound:\\n {.val {incomplete}}.\\n You can drop incomplete prediction intervals using `na.rm = TRUE`.\")) #> } #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> <bytecode: 0x5626cbe3e440> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"sample-based forecasts, default scoring rules : \"crps\" = crps_sample() \"overprediction\" = overprediction_sample() \"underprediction\" = underprediction_sample() \"dispersion\" = dispersion_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"# S3 method for class 'forecast_sample' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"x forecast object (validated data.table predicted observed values, see as_forecast_binary()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"get_metrics(example_sample_continuous, exclude = \"mad\") #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> <bytecode: 0x5626c5a06138> #> <environment: namespace:scoringutils> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> <bytecode: 0x5626c9da6bb8> #> <environment: namespace:scoringutils> #> #> $crps #> function (observed, predicted, separate_results = FALSE, ...) #> { #> assert_input_sample(observed, predicted) #> crps <- scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> if (separate_results) { #> medians <- apply(predicted, 1, median) #> dispersion <- scoringRules::crps_sample(y = medians, #> dat = predicted, ...) #> overprediction <- rep(0, length(observed)) #> underprediction <- rep(0, length(observed)) #> overprediction[observed < medians] <- scoringRules::crps_sample(y = observed[observed < #> medians], dat = predicted[observed < medians, , drop = FALSE], #> ...) #> underprediction[observed > medians] <- scoringRules::crps_sample(y = observed[observed > #> medians], dat = predicted[observed > medians, , drop = FALSE], #> ...) #> overprediction[overprediction > 0] <- overprediction[overprediction > #> 0] - dispersion[overprediction > 0] #> underprediction[underprediction > 0] <- underprediction[underprediction > #> 0] - dispersion[underprediction > 0] #> return(list(crps = crps, dispersion = dispersion, underprediction = underprediction, #> overprediction = overprediction)) #> } #> else { #> return(crps) #> } #> } #> <bytecode: 0x5626cd6a3b30> #> <environment: namespace:scoringutils> #> #> $overprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$overprediction) #> } #> <bytecode: 0x5626cd07aed8> #> <environment: namespace:scoringutils> #> #> $underprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$underprediction) #> } #> <bytecode: 0x5626cd07a3b0> #> <environment: namespace:scoringutils> #> #> $dispersion #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$dispersion) #> } #> <bytecode: 0x5626cd07d6b8> #> <environment: namespace:scoringutils> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> <bytecode: 0x5626cd07cb90> #> <environment: namespace:scoringutils> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> <bytecode: 0x5626cd47e540> #> <environment: namespace:scoringutils> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> <bytecode: 0x5626cd07f098> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get metrics — get_metrics","title":"Get metrics — get_metrics","text":"Generic function obtain default metrics available scoring metrics used scoring. called forecast object returns list functions can used scoring. called scores object (see score()), returns character vector names metrics used scoring. See documentation actual methods See Also section details. Alternatively call ?get_metrics.<forecast_type> ?get_metrics.scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get metrics — get_metrics","text":"","code":"get_metrics(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get metrics — get_metrics","text":"x forecast scores object. ... Additional arguments passed method.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics.scores","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"","code":"# S3 method for class 'scores' get_metrics(x, error = FALSE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"x scores object, (data.table attribute metrics produced score()). error Throw error attribute called metrics? Default FALSE. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"Character vector names scoring rules used scoring.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"Generate Probability Integral Transformation (PIT) histogram validated forecast objects. See examples plot result function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_pit_histogram(forecast, num_bins = NULL, breaks = NULL, by, ...) # S3 method for class 'forecast_sample' get_pit_histogram( forecast, num_bins = 10, breaks = NULL, by, integers = c(\"nonrandom\", \"random\", \"ignore\"), n_replicates = NULL, ... ) get_pit_histogram(forecast, num_bins, breaks, by, ...) # Default S3 method get_pit_histogram(forecast, num_bins, breaks, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"forecast forecast object (validated data.table predicted observed values). num_bins number bins PIT histogram. sample-based forecasts, default 10 bins. quantile-based forecasts, default one bin available quantile. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored. 0 1 always added left right bounds, respectively. Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics. integers handle integer forecasts (count data). based methods described Czado et al. (2007). \"nonrandom\" (default) function use non-randomised PIT method. \"random\", use randomised PIT method. \"ignore\", treat integer forecasts continuous. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous integers set random.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"data.table density values bin PIT histogram.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit_histogram.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation histogram — get_pit_histogram.forecast_quantile","text":"","code":"library(\"ggplot2\") example <- as_forecast_sample(example_sample_continuous) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit_histogram(example, by = \"model\") ggplot(result, aes(x = mid, y = density)) + geom_col() + facet_wrap(. ~ model) + labs(x = \"Quantile\", \"Density\") # example with quantile data example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit_histogram(example, by = \"model\") ggplot(result, aes(x = mid, y = density)) + geom_col() + facet_wrap(. ~ model) + labs(x = \"Quantile\", \"Density\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section additional information required input format.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-binary-point.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","title":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","text":"Illustration required inputs binary point forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for nominal forecasts — illustration-input-metric-nominal","title":"Illustration of required inputs for nominal forecasts — illustration-input-metric-nominal","text":"Illustration required inputs nominal forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","title":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","text":"Illustration required inputs quantile-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","title":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","text":"Illustration required inputs sample-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.7984288 0.6188294 0.7680146 0.7392137 2.4496739 0.6725337 0.5966822 #> [8] 0.6659581 0.6581834 1.2866256 0.7423596 0.6155816 0.6176548 0.6484069 #> [15] 0.6584252 0.6929587 0.7558694 0.6724925 0.5716195 0.6251716 0.7431658 #> [22] 0.6906088 0.6524924 0.6274762 1.8011311 0.5840689 0.6602319 0.7076748 #> [29] 0.6060337 0.5491802 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.6735755 0.6315584 0.6931931 2.4596254 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 1.1441141 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 1.6335484 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $dispersion #> [1] NA 0.6735755 0.6315584 0.6931931 0.6195326 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 0.5198196 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 0.4719967 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 1.8400928 0.0000000 0.0000000 #> [8] 0.0000000 0.0000000 0.6242945 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 0.0000000 0.0000000 1.1615517 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast_binary","title":"Test whether an object is a forecast object — is_forecast_binary","text":"Test whether object forecast object. can test specific forecast_<type> class using appropriate is_forecast_<type> function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"is_forecast_binary(x) is_forecast_nominal(x) is_forecast_point(x) is_forecast_quantile(x) is_forecast_sample(x) is_forecast(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast_binary","text":"x R object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast_binary","text":"is_forecast: TRUE object class forecast, FALSE otherwise. is_forecast_<type>*: TRUE object class forecast_* addition class forecast, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"forecast_binary <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"library(magrittr) # pipe operator log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 example_quantile[observed > 0, ] %>% as_forecast_quantile() %>% transform_forecasts(fun = log_shift, offset = 1) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> <char> <Date> <char> <num> <char> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> <Date> <num> <num> <char> <num> #> 1: <NA> NA NA <NA> NA #> 2: <NA> NA NA <NA> NA #> 3: <NA> NA NA <NA> NA #> 4: <NA> NA NA <NA> NA #> 5: <NA> NA NA <NA> NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> <char> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. log score negative logarithm predictive density evaluated observed value. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: function uses kernel density estimation, well defined integer-valued Monte Carlo Samples. See scoringRules package details alternatives, e.g. calculating scores specific discrete probability distributions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 2.415978 1.391162 2.876486 1.778651 1.990756 3.187023 2.735783 2.308400 #> [9] 2.217484 2.403781 2.688615 2.813455 2.178733 2.080475 2.668287 2.773460 #> [17] 2.619100 2.600304 2.610290 3.177565 3.065729 2.454865 2.860266 2.676266 #> [25] 3.299807 2.918963 2.538826 2.698623 2.858104 2.869216"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 4.4478 5.9304 5.9304 5.9304 5.9304 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments data.table::.data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { # \\dontrun{ df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group( scores, metric, baseline, compare = \"model\", by, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation for counts — pit_histogram_sample","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation for counts — pit_histogram_sample","text":"","code":"pit_histogram_sample( observed, predicted, quantiles, integers = c(\"nonrandom\", \"random\", \"ignore\"), n_replicates = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation for counts — pit_histogram_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. quantiles vector quantiles calculate PIT. integers handle integer forecasts (count data). based methods described Czado et al. (2007). \"nonrandom\" (default) function use non-randomised PIT method. \"random\", use randomised PIT method. \"ignore\", treat integer forecasts continuous. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous integers set random.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation for counts — pit_histogram_sample","text":"vector PIT histogram densities bins corresponding given quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete nonnegative outcomes incidence counts, PIT longer uniform even forecasts ideal. case two methods available ase described Czado et al. (2007). default, nonrandomised PIT calculated using conditional cumulative distribution function $$ F(u) = \\begin{cases} 0 & \\text{} v < P_t(k_t - 1) \\\\ (v - P_t(k_t - 1)) / (P_t(k_t) - P_t(k_t - 1)) & \\text{} P_t(k_t - 1) \\leq v < P_t(k_t) \\\\ 1 & \\text{} v \\geq P_t(k_t) \\end{cases} $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence \\(k\\) time \\(t\\) \\(P_t (-1) = 0\\) definition. Values PIT histogram created averaging \\(n\\) predictions, $$ $$ calculating value bin quantile \\(q_i\\) quantile \\(q_{+ 1}\\) $$ $$ Alternatively, randomised PIT can used instead. case, PIT $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1)) $$ \\(v\\) standard uniform independent \\(k\\). values PIT histogram calculated binning $u_t$ values .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation for counts — pit_histogram_sample","text":"Claudia Czado, Tilmann Gneiting Leonhard Held (2009) Predictive model assessment count data. Biometrika, 96(4), 633-648. Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_histogram_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation for counts — pit_histogram_sample","text":"","code":"## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_histogram_sample(observed, predicted, quantiles = seq(0, 1, 0.1)) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_histogram_sample(observed, predicted, quantiles = seq(0, 1, 0.1)) ## integer predictions, randomised PIT observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_histogram_sample( observed, predicted, quantiles = seq(0, 1, 0.1), integers = \"random\", n_replicates = 30 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations, digits = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations(). digits number indicating many decimal places correlations rounded . default (digits = NULL) rounding takes place.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- scores %>% summarise_scores() %>% get_correlations() plot_correlations(correlations, digits = 2)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) library(magrittr) # pipe operator forecast_counts <- example_quantile %>% as_forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"target_type\", \"target_end_date\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast","title":"Print information about a forecast object — print.forecast","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast","text":"","code":"# S3 method for class 'forecast' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast","text":"x forecast object ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast","text":"","code":"dat <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name #> <char> <Date> <char> <num> <char> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> <Date> <num> <int> <char> <num> #> 1: <NA> NA NA <NA> NA #> 2: <NA> NA NA <NA> NA #> 3: <NA> NA NA <NA> NA #> 4: <NA> NA NA <NA> NA #> 5: <NA> NA NA <NA> NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast_quantile()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper/helper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score.forecast_binary","title":"Evaluate forecasts — score.forecast_binary","text":"score() applies selection scoring metrics forecast object. score() generic dispatches different methods depending class input data. See as_forecast_binary(), as_forecast_quantile() etc. information create forecast object. See get_forecast_unit() information concept forecast unit. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_nominal' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_point' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = get_metrics(forecast), ...) score(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score.forecast_binary","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Evaluate forecasts — score.forecast_binary","text":"Customising metrics want pass arguments scoring function, need change scoring function via e.g. purrr::partial() pass updated list functions custom metric metrics argument score(). example, use interval_coverage() interval_range = 90, define new function, e.g. interval_coverage_90 <- purrr::partial(interval_coverage, interval_range = 90) pass new function metrics score(). Note want pass variable argument, can unquote !! make sure value evaluated function created. Consider following example:","code":"custom_arg <- \"foo\" print1 <- purrr::partial(print, x = custom_arg) print2 <- purrr::partial(print, x = !!custom_arg) custom_arg <- \"bar\" print1() # prints 'bar' print2() # prints 'foo'"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score.forecast_binary","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score.forecast_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"library(magrittr) # pipe operator validated <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> <num> <num> <num> <num> <num> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 24101.07031 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 38473.60156 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 27923.81250 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 53.13281 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 233.25781 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 78.47656 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast_quantile( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> <char> <Date> <char> <num> <char> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> <num> <num> <num> <num> <num> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 ae_median #> <lgcl> <lgcl> <num> #> 1: TRUE TRUE 12271 #> 2: FALSE FALSE 25620 #> 3: FALSE TRUE 44192 #> 4: TRUE TRUE 14 #> 5: TRUE TRUE 15 #> --- #> 883: TRUE TRUE 53 #> 884: TRUE TRUE 1 #> 885: FALSE TRUE 46 #> 886: TRUE TRUE 26 #> 887: FALSE TRUE 108 # forecast formats with different metrics if (FALSE) { # \\dontrun{ score(as_forecast_binary(example_binary)) score(as_forecast_quantile(example_quantile)) score(as_forecast_point(example_point)) score(as_forecast_sample(example_sample_discrete)) score(as_forecast_sample(example_sample_continuous)) } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal last/highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 0.169810872 0.319983015 0.720333826 0.136189585 0.539816982 0.573052550 #> [7] 0.210298251 0.005442733 0.940506004 0.209025175 0.132818958 0.775957259 #> [13] 0.533583639 0.773922330 0.177404878 0.032952811 0.700006942 0.860101989 #> [19] 0.010349776 0.035994321 0.423125196 0.650581640 0.408156615 0.150229898 #> [25] 0.003399837 0.300913133 0.070701554 0.002211134 0.008862522 0.099258707 logs_binary(observed, predicted) #> [1] 0.53116635 0.83395161 1.88865474 0.46051080 1.32697839 1.41470349 #> [7] 0.61356527 0.07663796 3.49981031 0.61100092 0.45325406 2.12766051 #> [13] 1.31106851 2.11800405 0.54678893 0.20031742 1.81194692 2.62302230 #> [19] 0.10728887 0.21037750 1.05119662 1.64292444 1.01852104 0.49036148 #> [25] 0.06007715 0.79530283 0.30910680 0.04816419 0.09887158 0.37841454"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Log score for nominal outcomes — logs_nominal","title":"Log score for nominal outcomes — logs_nominal","text":"Log score nominal outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"logs_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log score for nominal outcomes — logs_nominal","text":"observed factor length n N levels holding observed values. predicted nxN matrix predictive probabilities, n (number rows) number observations N (number columns) number possible outcomes. predicted_label factor length N, denoting outcome probabilities predicted correspond .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log score for nominal outcomes — logs_nominal","text":"numeric vector size n log scores","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"factor_levels <- c(\"one\", \"two\", \"three\") predicted_label <- factor(c(\"one\", \"two\", \"three\"), levels = factor_levels) observed <- factor(c(\"one\", \"three\", \"two\"), levels = factor_levels) predicted <- matrix(c(0.8, 0.1, 0.4, 0.1, 0.2, 0.4, 0.1, 0.7, 0.2), nrow = 3) logs_nominal(observed, predicted, predicted_label) #> [1] 0.2231436 0.3566749 0.9162907"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Facilitate evaluation forecasts convenient framework based data.table. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper, doi:10.48550/arXiv.2205.07090 .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor] Michael Chirico michaelchirico4@gmail.com (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 1.120954e+00 5.506171e-01 8.158564e+00 4.953364e-02 1.978748e-01 #> [6] 3.924656e+00 4.315103e-03 1.971522e+00 5.325623e-03 3.438576e+00 #> [11] 4.944164e-02 9.564610e-04 6.898922e-01 3.784738e+00 1.028803e-01 #> [16] 2.632511e-05 3.644782e-01 6.029127e-01 8.747270e-01 2.939886e-01 #> [21] 2.574917e-04 5.380933e+00 1.563555e+00 1.536483e+00 1.565979e-02 #> [26] 1.691586e-01 1.467380e+01 2.184548e+00 1.223069e+00 1.486889e-01"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = get_metrics(example_binary), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x5626cc441010> #> <environment: namespace:scoringutils> #> select_metrics( metrics = get_metrics(example_binary), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> <bytecode: 0x5626cc441010> #> <environment: namespace:scoringutils> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called constructing forecast object via forecast_unit argument as_forecast_<type>. done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section additional information required input format. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"library(magrittr) # pipe operator example_quantile %>% scoringutils:::set_forecast_unit( c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, horizon, and model #> #> Key: <location, target_end_date, target_type> #> observed quantile_level predicted location target_end_date target_type #> <num> <num> <int> <char> <Date> <char> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> <num> <char> #> 1: NA <NA> #> 2: NA <NA> #> 3: NA <NA> #> 4: NA <NA> #> 5: NA <NA> #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", fun = mean, ...) summarize_scores(scores, by = \"model\", fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default \"model\", .e. scores summarised \"model\" column. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"library(magrittr) # pipe operator scores <- example_sample_continuous %>% as_forecast_sample() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores, by = \"model\") #> model bias dss crps overprediction #> <char> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 5281.81845 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 6701.83844 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 6986.16572 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 11.20505 #> underprediction dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> <num> #> 1: 2433.20525 2161.93515 10.747811 8763.6176 12406.03100 2.103026e+09 #> 2: 5864.53649 2743.31134 Inf 9680.3792 18932.50196 2.885063e+09 #> 3: 1740.93388 3174.33393 Inf 12999.5404 14680.12285 3.152268e+09 #> 4: 19.28195 29.70318 5.941622 123.6211 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 #> overprediction underprediction dispersion log_score mad ae_median #> <num> <num> <num> <num> <num> <num> #> 1: 10552.97603 4861.015121 4289.06407 15.633420 17385.2629 24749.39707 #> 2: 13346.03509 11727.330575 5380.21780 Inf 18982.2128 37648.01693 #> 3: 13462.90822 3346.583024 6087.02483 Inf 24929.3438 28233.04536 #> 4: 10.66088 5.395380 34.80623 5.862203 141.9723 62.66492 #> 5: 57.64179 1.742401 106.40489 6.977391 378.5457 216.98699 #> 6: 11.20505 19.281946 29.70318 5.941622 123.6211 79.66001 #> 7: 19.58555 13.849095 41.35549 6.024092 167.4829 102.18939 #> se_mean #> <num> #> 1: 4.206042e+09 #> 2: 5.769964e+09 #> 3: 6.082863e+09 #> 4: 1.080233e+04 #> 5: 1.622417e+05 #> 6: 1.371418e+04 #> 7: 3.243111e+04 # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps overprediction #> <char> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 37275.23950 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 39070.74445 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 40776.81690 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 21.34675 #> underprediction dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> <num> #> 1: 8634.87723 5163.42293 21.510119 19799.1620 42801.64123 1.564286e+10 #> 2: 20537.03929 3664.87255 NaN 13610.4174 49458.36446 1.760651e+10 #> 3: 8096.39644 7266.11787 NaN 29616.1714 51129.54601 2.209086e+10 #> 4: 36.98584 29.60927 1.126019 123.3465 76.09471 2.994664e+04 # round digits summarise_scores(scores, by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps overprediction underprediction #> <char> <num> <num> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 5300 2400 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 6700 5900 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 7000 1700 #> 4: UMass-MechBayes -0.0270 10 60 11 19 #> dispersion log_score mad ae_median se_mean #> <num> <num> <num> <num> <num> #> 1: 2200 11.0 8800 12000 2.1e+09 #> 2: 2700 Inf 9700 19000 2.9e+09 #> 3: 3200 Inf 13000 15000 3.2e+09 #> 4: 30 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values). fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754302 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419360 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950677 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718827 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609898 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677099 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast_quantile() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754302 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419360 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950677 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718827 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609898 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677099 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: <location, target_end_date, target_type> #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 11.754310 Germany <NA> #> 2: DE 2021-01-02 Deaths 8.419580 Germany <NA> #> 3: DE 2021-01-09 Cases 11.950683 Germany <NA> #> 4: DE 2021-01-09 Deaths 8.718991 Germany <NA> #> 5: DE 2021-01-16 Cases 11.609907 Germany <NA> #> 6: DE 2021-01-16 Deaths 8.677269 Germany <NA> #> quantile_level predicted model horizon #> <num> <num> <char> <num> #> 1: NA NA <NA> NA #> 2: NA NA <NA> NA #> 3: NA NA <NA> NA #> 4: NA NA <NA> NA #> 5: NA NA <NA> NA #> 6: NA NA <NA> NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> <char> <char> <num> <num> <num> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> <num> <num> <num> <num> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> ae_median #> <num> #> 1: 8880.542969 #> 2: 16156.871094 #> 3: 11208.072874 #> 4: 78.476562 #> 5: 22.458900 #> 6: 39.185406 #> 7: 25.585018 #> 8: 2.069103 # adding multiple transformations example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type observed location_name forecast_date #> <char> <Date> <char> <num> <char> <Date> #> 1: DE 2021-01-02 Cases 127300 Germany <NA> #> 2: DE 2021-01-02 Deaths 4534 Germany <NA> #> 3: DE 2021-01-09 Cases 154922 Germany <NA> #> 4: DE 2021-01-09 Deaths 6117 Germany <NA> #> 5: DE 2021-01-16 Cases 110183 Germany <NA> #> 6: DE 2021-01-16 Deaths 5867 Germany <NA> #> quantile_level predicted model horizon scale #> <num> <num> <char> <num> <char> #> 1: NA NA <NA> NA natural #> 2: NA NA <NA> NA natural #> 3: NA NA <NA> NA natural #> 4: NA NA <NA> NA natural #> 5: NA NA <NA> NA natural #> 6: NA NA <NA> NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). set \\(k = 1, \\dots, K\\) prediction intervals median \\(m\\), can compute weighted interval score (WIS) sum interval scores individual intervals: $$ \\text{WIS}_{\\alpha_{\\{0:K\\}}}(F, y) = \\frac{1}{K + 1/2} \\times \\left(w_0 \\times |y - m| + \\sum_{k=1}^{K} \\left\\{ w_k \\times \\text{}_{\\alpha_k}(F, y) \\right\\}\\right) $$ individual scores usually weighted \\(w_k = \\frac{\\alpha_k}{2}\\). weight ensures increasing number equally spaced quantiles, WIS converges continuous ranked probability score (CRPS). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE ) dispersion_quantile(observed, predicted, quantile_level, ...) overprediction_quantile(observed, predicted, quantile_level, ...) underprediction_quantile(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction_quantile(), underprediction_quantile() dispersion_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion_quantile(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation)","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Weighted interval score (WIS) — wis","text":"Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new set as_forecast_<type>() functions introduced validate data convert inputs forecast object (data.table forecast class additional class corresponding forecast type (see )). Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile, forecast_sample, forecast_nominal, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using functions as_forecast_point(), as_forecast_binary(), as_forecast_quantile(), as_forecast_sample() (replace previous check_forecast()). forecast object data.table class forecast additional class corresponding forecast type (e.g. forecast_quantile). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. binary point forecasts, columns “observed” “predicted” now removed consistency forecast types. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using function get_metrics(), generic S3 methods forecast type. returns named list scoring rules suitable respective forecast object. example, call get_metrics(example_quantile). Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using purrr::partial() select_metric(). See ?score() information. CRPS now reported decomposition dispersion, overprediction underprediction. functionality calculate Probability Integral Transform (PIT) deprecated replaced functionality calculate PIT histograms, using get_pit_histogram() function; part change, nonrandomised PITs can now calculated count data, done default","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"as_forecast_<type>() functions create forecast object validates . also allow users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases. See ?as_forecast() information. as_forecast_<type>() functions like e.g. as_forecast_point() as_forecast_quantile() S3 methods converting another forecast type respective forecast type. example, as_forecast_quantile() method converting forecast_sample object forecast_quantile object estimating quantiles samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast_quantile( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast_<type>(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, users can call `as_forecast_<type>() re-validate forecast object. Simply printing object also provide additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast_quantile() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts(). Renamed interval_coverage_quantile() interval_coverage(). “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se.get_duplicate_forecasts() now sorts outputs according forecast unit, making easier spot duplicates. addition, counts option allows user display number duplicates forecast unit, rather raw duplicated rows. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring. Function set_forecast_unit() deleted. Instead now forecast_unit argument as_forecast_<type>() well get_duplicate_forecasts(). Removed interval_coverage_dev_quantile(). Users can still access difference nominal actual interval coverage using get_coverage(). pit(), pit_sample() plot_pit() removed replaced functionality create PIT histograms (pit_histogram_sampel() get_pit_histogram())","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles. contrast previous correlation function, get_correlations doesn’t round correlations default. Instead, plot_correlations now digits argument allows users round correlations plotting . Alternatively, using dplyr, call something like mutate(correlations, across((.numeric), \\(x) signif(x, digits = 2))) output get_correlations. wis() now errors default quantile levels form valid prediction intervals returns NA missing values. Previously, na.rm set TRUE default, lead unexpected results, users aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs. scoringutils now requires R >= 4.0","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug as_forecast_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}] diff --git a/dev/sitemap.xml b/dev/sitemap.xml index df5e10766..88cf67767 100644 --- a/dev/sitemap.xml +++ b/dev/sitemap.xml @@ -15,8 +15,8 @@ <url><loc>https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html</loc></url> -<url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html</loc></url> +<url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast_doc_template.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html</loc></url> @@ -83,6 +83,7 @@ <url><loc>https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/get_type.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-binary-point.html</loc></url> +<url><loc>https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-nominal.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-quantile.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-sample.html</loc></url> <url><loc>https://epiforecasts.io/scoringutils/dev/reference/index.html</loc></url>