From 93bcd9198a90b42dd1185367eef40fd7f5ba256a Mon Sep 17 00:00:00 2001 From: nikosbosse <37978797+nikosbosse@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:10:16 +0000 Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20@=20epiforec?= =?UTF-8?q?asts/scoringutils@413d4272ba58a723a667d984fd949702800fd818=20?= =?UTF-8?q?=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PULL_REQUEST_TEMPLATE.html | 105 ++ articles/metric-details.html | 119 +- articles/scoring-forecasts-directly.html | 136 +- articles/scoringutils.html | 948 +++++++----- .../figure-html/unnamed-chunk-18-1.png | Bin 93680 -> 86400 bytes .../figure-html/unnamed-chunk-24-1.png | Bin 0 -> 114557 bytes .../figure-html/unnamed-chunk-25-1.png | Bin 0 -> 114447 bytes .../figure-html/unnamed-chunk-30-1.png | Bin 105408 -> 79482 bytes .../figure-html/unnamed-chunk-31-1.png | Bin 0 -> 111786 bytes .../figure-html/unnamed-chunk-6-1.png | Bin 69976 -> 69999 bytes index.html | 138 +- news/index.html | 95 +- pkgdown.yml | 2 +- reference/Rplot003.png | Bin 13436 -> 13433 bytes reference/abs_error.html | 38 +- reference/add_coverage.html | 188 +-- reference/add_pairwise_comparison.html | 141 -- reference/ae_median_quantile.html | 72 +- reference/ae_median_sample.html | 34 +- reference/assert_dims_ok_point.html | 138 -- reference/assert_input_binary.html | 115 -- reference/assert_input_interval.html | 120 -- reference/assert_input_quantile.html | 123 -- reference/assert_input_sample.html | 112 -- reference/assign_attributes.html | 106 -- reference/assure_model_column.html | 104 -- reference/avail_forecasts.html | 69 +- reference/available_forecasts.html | 145 -- reference/bias_quantile.html | 45 +- reference/bias_range.html | 24 +- reference/bias_sample.html | 40 +- ...sert_input_point.html => brier_score.html} | 56 +- reference/check_attribute_conflict.html | 120 -- reference/check_columns_present.html | 110 -- reference/check_data_columns.html | 106 -- reference/check_dims_ok_point.html | 138 -- reference/check_duplicates.html | 106 -- ...al_length.html => check_equal_length.html} | 20 +- reference/check_forecasts.html | 297 ++++ reference/check_has_attribute.html | 104 -- reference/check_input_binary.html | 115 -- reference/check_input_interval.html | 120 -- reference/check_input_point.html | 109 -- reference/check_input_quantile.html | 118 -- ..._single_vector.html => check_metrics.html} | 40 +- reference/check_no_NA_present.html | 110 -- ...sert_not_null.html => check_not_null.html} | 8 +- reference/check_number_per_forecast.html | 110 -- reference/check_numeric_vector.html | 164 -- reference/check_predictions.html | 116 ++ reference/check_quantiles.html | 4 +- reference/check_summary_params.html | 22 + .../{validate.html => check_true_values.html} | 91 +- reference/check_try.html | 109 -- reference/collapse_messages.html | 8 +- reference/compare_two_models.html | 5 +- reference/correlation.html | 46 +- reference/crps_sample.html | 31 +- reference/delete_columns.html | 4 +- reference/document_assert_functions.html | 89 -- reference/document_check_functions.html | 100 -- reference/document_score_data.html | 93 -- reference/document_test_functions.html | 88 -- reference/dss_sample.html | 33 +- reference/example_binary.html | 8 +- reference/example_continuous.html | 10 +- reference/example_integer.html | 12 +- reference/example_point.html | 11 +- reference/example_quantile.html | 8 +- .../example_quantile_forecasts_only.html | 4 +- reference/example_truth_only.html | 6 +- reference/figures/unnamed-chunk-4-1.png | Bin 66831 -> 68823 bytes reference/filter_function_args.html | 116 -- ...te_forecasts.html => find_duplicates.html} | 110 +- ...recast_type.html => geom_mean_helper.html} | 54 +- reference/geometric_mean.html | 103 -- reference/get_forecast_unit.html | 49 +- reference/get_metrics.html | 102 -- ..._present.html => get_prediction_type.html} | 49 +- reference/get_protected_columns.html | 26 +- reference/get_range_from_quantile.html | 117 -- reference/get_scoringutils_attributes.html | 99 -- reference/get_target_type.html | 122 ++ reference/index.html | 367 +---- reference/infer_rel_skill_metric.html | 9 +- reference/interval_coverage.html | 157 -- reference/interval_coverage_dev_quantile.html | 216 --- reference/interval_score.html | 98 +- ..._point.html => is_scoringutils_check.html} | 42 +- reference/log_shift.html | 28 +- ...t_type_is_binary.html => logs_binary.html} | 46 +- reference/logs_sample.html | 32 +- reference/mad_sample.html | 30 +- reference/make_NA.html | 42 +- reference/merge_pred_and_obs.html | 46 +- reference/metrics_binary.html | 100 -- reference/metrics_quantile.html | 127 -- reference/metrics_sample.html | 118 -- reference/new_scoringutils.html | 117 -- reference/pairwise_comparison.html | 11 +- reference/pairwise_comparison_one_group.html | 5 +- reference/permutation_test.html | 53 +- reference/pit.html | 8 +- reference/pit_sample-1.png | Bin 19346 -> 18078 bytes reference/pit_sample-2.png | Bin 17096 -> 17096 bytes reference/pit_sample.html | 30 +- ...lot.scoringutils_available_forecasts-1.png | Bin 49283 -> 0 bytes ...plot.scoringutils_available_forecasts.html | 146 -- reference/plot_avail_forecasts-1.png | Bin 0 -> 50597 bytes reference/plot_avail_forecasts.html | 35 +- reference/plot_correlation-1.png | Bin 105530 -> 79800 bytes reference/plot_correlation.html | 5 +- reference/plot_heatmap-1.png | Bin 43403 -> 40570 bytes reference/plot_heatmap.html | 4 +- reference/plot_interval_coverage-1.png | Bin 0 -> 114767 bytes reference/plot_interval_coverage.html | 14 +- reference/plot_pairwise_comparison-1.png | Bin 78079 -> 78098 bytes reference/plot_pairwise_comparison.html | 2 + reference/plot_pit-1.png | Bin 17048 -> 19473 bytes reference/plot_pit-3.png | Bin 37949 -> 37975 bytes reference/plot_pit.html | 14 +- reference/plot_quantile_coverage-1.png | Bin 0 -> 116043 bytes reference/plot_quantile_coverage.html | 10 +- reference/plot_ranges-1.png | Bin 0 -> 105943 bytes reference/plot_ranges-2.png | Bin 0 -> 76560 bytes reference/plot_ranges.html | 20 +- reference/plot_score_table-1.png | Bin 134407 -> 119487 bytes reference/plot_score_table-2.png | Bin 132283 -> 117890 bytes reference/plot_score_table.html | 6 +- reference/plot_wis.html | 2 + ...table.html => prediction_is_quantile.html} | 26 +- reference/print.scoringutils_check.html | 90 +- reference/quantile_score.html | 30 +- reference/quantile_to_interval.html | 189 --- ...ample.html => quantile_to_range_long.html} | 35 +- reference/remove_scoringutils_class.html | 102 -- reference/run_safely.html | 139 -- reference/sample_to_quantile.html | 24 +- reference/score.html | 1328 ++++++++++++++--- reference/score_binary.html | 132 ++ reference/score_quantile.html | 173 +++ .../{get_type.html => score_sample.html} | 74 +- reference/scoring-functions-binary.html | 192 --- reference/scoringutils-package.html | 18 +- reference/se_mean_sample.html | 45 +- reference/set_forecast_unit.html | 84 +- reference/squared_error.html | 31 +- reference/strip_attributes.html | 107 -- reference/summarise_scores.html | 494 +++--- reference/test_columns_present.html | 109 -- reference/test_forecast_type_is_point.html | 99 -- reference/test_forecast_type_is_quantile.html | 99 -- reference/test_forecast_type_is_sample.html | 99 -- reference/transform_forecasts.html | 194 +-- reference/validate_general.html | 124 -- reference/validate_metrics.html | 110 -- reference/wis.html | 340 ----- search.json | 2 +- sitemap.xml | 191 +-- 159 files changed, 4445 insertions(+), 9128 deletions(-) create mode 100644 PULL_REQUEST_TEMPLATE.html create mode 100644 articles/scoringutils_files/figure-html/unnamed-chunk-24-1.png create mode 100644 articles/scoringutils_files/figure-html/unnamed-chunk-25-1.png create mode 100644 articles/scoringutils_files/figure-html/unnamed-chunk-31-1.png delete mode 100644 reference/add_pairwise_comparison.html delete mode 100644 reference/assert_dims_ok_point.html delete mode 100644 reference/assert_input_binary.html delete mode 100644 reference/assert_input_interval.html delete mode 100644 reference/assert_input_quantile.html delete mode 100644 reference/assert_input_sample.html delete mode 100644 reference/assign_attributes.html delete mode 100644 reference/assure_model_column.html delete mode 100644 reference/available_forecasts.html rename reference/{assert_input_point.html => brier_score.html} (59%) delete mode 100644 reference/check_attribute_conflict.html delete mode 100644 reference/check_columns_present.html delete mode 100644 reference/check_data_columns.html delete mode 100644 reference/check_dims_ok_point.html delete mode 100644 reference/check_duplicates.html rename reference/{assert_equal_length.html => check_equal_length.html} (52%) create mode 100644 reference/check_forecasts.html delete mode 100644 reference/check_has_attribute.html delete mode 100644 reference/check_input_binary.html delete mode 100644 reference/check_input_interval.html delete mode 100644 reference/check_input_point.html delete mode 100644 reference/check_input_quantile.html rename reference/{bias_quantile_single_vector.html => check_metrics.html} (73%) delete mode 100644 reference/check_no_NA_present.html rename reference/{assert_not_null.html => check_not_null.html} (64%) delete mode 100644 reference/check_number_per_forecast.html delete mode 100644 reference/check_numeric_vector.html create mode 100644 reference/check_predictions.html rename reference/{validate.html => check_true_values.html} (51%) delete mode 100644 reference/check_try.html delete mode 100644 reference/document_assert_functions.html delete mode 100644 reference/document_check_functions.html delete mode 100644 reference/document_score_data.html delete mode 100644 reference/document_test_functions.html delete mode 100644 reference/filter_function_args.html rename reference/{get_duplicate_forecasts.html => find_duplicates.html} (58%) rename reference/{get_forecast_type.html => geom_mean_helper.html} (70%) delete mode 100644 reference/geometric_mean.html delete mode 100644 reference/get_metrics.html rename reference/{test_columns_not_present.html => get_prediction_type.html} (65%) delete mode 100644 reference/get_range_from_quantile.html delete mode 100644 reference/get_scoringutils_attributes.html create mode 100644 reference/get_target_type.html delete mode 100644 reference/interval_coverage.html delete mode 100644 reference/interval_coverage_dev_quantile.html rename reference/{metrics_point.html => is_scoringutils_check.html} (75%) rename reference/{test_forecast_type_is_binary.html => logs_binary.html} (62%) delete mode 100644 reference/metrics_binary.html delete mode 100644 reference/metrics_quantile.html delete mode 100644 reference/metrics_sample.html delete mode 100644 reference/new_scoringutils.html delete mode 100644 reference/plot.scoringutils_available_forecasts-1.png delete mode 100644 reference/plot.scoringutils_available_forecasts.html create mode 100644 reference/plot_avail_forecasts-1.png create mode 100644 reference/plot_interval_coverage-1.png create mode 100644 reference/plot_quantile_coverage-1.png create mode 100644 reference/plot_ranges-1.png create mode 100644 reference/plot_ranges-2.png rename reference/{ensure_data.table.html => prediction_is_quantile.html} (51%) delete mode 100644 reference/quantile_to_interval.html rename reference/{check_input_sample.html => quantile_to_range_long.html} (74%) delete mode 100644 reference/remove_scoringutils_class.html delete mode 100644 reference/run_safely.html create mode 100644 reference/score_binary.html create mode 100644 reference/score_quantile.html rename reference/{get_type.html => score_sample.html} (57%) delete mode 100644 reference/scoring-functions-binary.html delete mode 100644 reference/strip_attributes.html delete mode 100644 reference/test_columns_present.html delete mode 100644 reference/test_forecast_type_is_point.html delete mode 100644 reference/test_forecast_type_is_quantile.html delete mode 100644 reference/test_forecast_type_is_sample.html delete mode 100644 reference/validate_general.html delete mode 100644 reference/validate_metrics.html delete mode 100644 reference/wis.html diff --git a/PULL_REQUEST_TEMPLATE.html b/PULL_REQUEST_TEMPLATE.html new file mode 100644 index 000000000..6c60dae88 --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.html @@ -0,0 +1,105 @@ + +NA • scoringutils + Skip to contents + + +
+
+
+

NA

+ Source: .github/PULL_REQUEST_TEMPLATE.md +
+ + +
+

Description

+

This PR closes #.

+

[Describe the changes that you made in this pull request.]

+
+
+

Checklist

+
  • +My PR is based on a package issue and I have explicitly linked it.
    • +
  • +I have included the target issue or issues in the PR title as follows: issue-number: PR title
    • +
  • +I have tested my changes locally.
    • +
  • +I have added or updated unit tests where necessary.
    • +
  • +I have updated the documentation if required.
    • +
  • +I have built the package locally and run rebuilt docs using roxygen2.
    • +
  • +My code follows the established coding standards and I have run lintr::lint_package() to check for style issues introduced by my changes.
    • +
  • +I have added a news item linked to this PR.
    • +
  • +I have reviewed CI checks for this PR and addressed them as far as I am able.
    • +
+ + +
+ + +
+ + + + + + + diff --git a/articles/metric-details.html b/articles/metric-details.html index 70eecb39c..74dcd7480 100644 --- a/articles/metric-details.html +++ b/articles/metric-details.html @@ -154,46 +154,6 @@

-Absolute error - - -y - - -y - - -n - - -y - - -Suitable for scoring the median of a predictive distribution - - - - -Squared error - - -y - - -y - - -n - - -y - - -Suitable for scoring the mean of a predictive distribution. - - - - Squared error @@ -251,9 +211,9 @@

-Proper scoring rule, smaller is better, only evaluates predictive -density at observed value (local), penalises over-confidence severely, -susceptible to outliers +Proper scoring rule, smaller is better, equals negative log of the +predictive density at observed value (local), penalises over-confidence +severely, susceptible to outliers @@ -447,13 +407,13 @@

-Under-, Over-prediction +Probability integral transform (PIT) -n +y -n +y n @@ -462,51 +422,11 @@

-Absolute amount of over-or under-prediction (components of WIS) - - - - -Probability integral transform (PIT) - - -y - - -y - - -n - - -y - - PIT transform is the CDF of the predictive distribution evaluated at the observed values. PIT values should be uniform. - -Dispersion - - -n - - -n - - -n - - -y - - -Dispersion component of WIS, measures width of predictive intervals. - - - Bias @@ -624,10 +544,10 @@

Detaile |X - X'| - \mathbb{E}_P |X - y|,\] where \(X\) and \(X'\) are independent realisations from the predictive distributions \(F\) with finite first moment and \(y\) is the -observed value. In this representation we can simply replace \(X\) and \(X'\) by samples sum over all possible -combinations to obtain the CRPS. For integer-valued forecasts, the RPS -is given as \[ \text{RPS}(F, y) = \sum_{x = -0}^\infty (F(x) - 1(x \geq y))^2. \]

+observed value. In this representation we can simply replace \(X\) and \(X'\) by samples and sum over all +possible combinations to obtain the CRPS. For integer-valued forecasts, +the RPS is given as \[ \text{RPS}(F, y) = +\sum_{x = 0}^\infty (F(x) - 1(x \geq y))^2. \]

Usage and caveats: Smaller values are better. The crps is a good choice for most practical purposes that involve decision making, as it takes the entire predictive distribution into account. If @@ -645,19 +565,18 @@

Detaile Log score -

The Log score is a proper scoring rule that is simply computed as the -log of the predictive density evaluated at the observed value. It is -given as \[ \text{log score} = \log f(y), -\] where \(f\) is the predictive -density function and y is the observed value. For integer-valued -forecasts, the log score can be computed as \[ \text{log score} = \log p_y, \] where +

The Log score is a proper scoring rule that is computed as the +negative log of the predictive density evaluated at the observed value. +It is given as \[ \text{log score} = -\log +f(y), \] where \(f\) is the +predictive density function and y is the observed value. For +integer-valued forecasts, the log score can be computed as \[ \text{log score} = -\log p_y, \] where \(p_y\) is the probability assigned to outcome p by the forecast F.

-Usage and caveats: Larger values are better, but +Usage and caveats: Smaller values are better, but sometimes the sign is reversed. The log score is sensitive to outliers, -as individual negative log score contributions quickly can become very -large if the event falls in the tails of the predictive distribution, -where \(f(y)\) (or \(p_y\)) is close to zero. Whether or not +as individual log score contributions can become very large if the event +falls in a range of the predictive distribution where \(f(y)\) (or \(p_y\)) is close to zero. Whether or not that is desirable depends ont the application. In scoringutils, the log score cannot be used for integer-valued forecasts, as the implementation requires a predictive density. In contrast to the crps, the log score is diff --git a/articles/scoring-forecasts-directly.html b/articles/scoring-forecasts-directly.html index c507f609d..5b2cc9c9d 100644 --- a/articles/scoring-forecasts-directly.html +++ b/articles/scoring-forecasts-directly.html @@ -100,8 +100,8 @@

Bias

For continuous forecasts, Bias is measured as \[B_t (P_t, x_t) = 1 - 2 \cdot (P_t (x_t))\]

where \(P_t\) is the empirical -cumulative distribution function of the prediction for the observed -value \(x_t\). Computationally, \(P_t (x_t)\) is just calculated as the +cumulative distribution function of the prediction for the true value +\(x_t\). Computationally, \(P_t (x_t)\) is just calculated as the fraction of predictive samples for \(x_t\) that are smaller than \(x_t\).

For integer valued forecasts, Bias is measured as

\[B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t @@ -110,20 +110,20 @@

Bias Bias can assume values between -1 and 1 and is 0 ideally.

 ## integer valued forecasts
-observed <- rpois(30, lambda = 1:30)
-predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
-bias_sample(observed, predicted)
-#>  [1] -0.150 -0.640 -0.815  0.680 -0.085  0.815 -0.260  0.060 -0.265 -0.660
-#> [11]  0.650 -0.185  0.740  0.195 -0.420 -0.535  0.385  0.305  0.160  0.145
-#> [21]  0.000  0.115 -0.670  0.420 -0.300  0.590 -0.165  0.435  0.375 -0.935
+true_values <- rpois(30, lambda = 1:30)
+predictions <- replicate(200, rpois(n = 30, lambda = 1:30))
+bias_sample(true_values, predictions)
+#>  [1] -0.080 -0.005 -0.735  0.025 -0.030 -0.425  0.500  0.835  0.390  0.205
+#> [11]  0.710 -0.205 -0.105 -0.200  0.845 -0.920  0.045 -0.730 -0.325  0.145
+#> [21]  0.905 -0.365  0.285 -0.830 -0.535  0.120 -0.430 -0.825 -0.805  0.425
 
 ## continuous forecasts
-observed <- rnorm(30, mean = 1:30)
-predicted <- replicate(200, rnorm(30, mean = 1:30))
-bias_sample(observed, predicted)
-#>  [1] -0.87  0.87  0.22  0.06  0.41  0.07 -0.28  0.34 -0.01  1.00 -0.65  0.61
-#> [13]  0.45  0.33 -0.15 -0.78  0.99  0.34 -0.50 -0.43  0.60  0.13  0.76 -0.05
-#> [25]  0.18 -0.37 -0.94 -0.06 -0.51  0.81
+true_values <- rnorm(30, mean = 1:30) +predictions <- replicate(200, rnorm(30, mean = 1:30)) +bias_sample(true_values, predictions) +#> [1] -0.74 -0.53 0.69 0.46 0.30 0.02 -0.63 0.89 -0.89 0.41 -0.08 0.83 +#> [13] 0.93 0.76 -0.93 0.46 -0.56 0.54 0.28 0.57 -0.82 -0.07 0.53 0.13 +#> [25] -0.05 -0.64 0.16 0.85 0.57 -0.98

Sharpness @@ -132,15 +132,15 @@

Sharpness?stats::mad

+single true value is measured as the normalised median of the absolute +deviation from the median of the predictive samples. For details, see +?stats::mad

-predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
-mad_sample(predicted = predicted)
-#>  [1] 1.4826 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652
-#> [11] 2.9652 4.4478 2.9652 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478 4.4478
-#> [21] 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.9304 4.4478 4.4478 4.4478
+predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) +mad_sample(predictions) +#> [1] 1.4826 1.4826 1.4826 1.4826 1.4826 2.2239 2.9652 2.9652 2.9652 2.9652 +#> [11] 2.9652 2.9652 3.7065 3.7065 3.7065 2.9652 2.9652 4.4478 4.4478 4.4478 +#> [21] 4.4478 5.1891 4.4478 4.4478 4.4478 5.9304 5.1891 4.4478 4.4478 4.4478

Calibration @@ -194,13 +194,13 @@

Continuous Ranked Probability used for continuous as well as integer valued forecasts. Smaller values are better.

-observed <- rpois(30, lambda = 1:30)
-predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
-crps_sample(observed, predicted)
-#>  [1] 0.598850 0.330325 0.805400 0.727300 1.006800 0.571475 0.783550 2.326300
-#>  [9] 0.757300 1.194825 0.900300 0.847800 3.568025 1.216400 1.005575 1.568925
-#> [17] 2.105850 4.121650 2.763150 9.077400 1.695000 1.442325 2.822500 8.750775
-#> [25] 6.054200 1.739825 8.494275 1.889400 7.316350 3.049750
+true_values <- rpois(30, lambda = 1:30) +predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) +crps_sample(true_values, predictions) +#> [1] 0.202275 0.272700 0.380850 0.528225 0.647450 1.160200 1.529450 0.728475 +#> [9] 1.109025 2.488150 6.495875 2.549200 1.351500 1.236150 1.479425 2.543850 +#> [17] 2.862000 1.859375 1.444675 1.210025 1.860700 1.950775 1.557475 2.633125 +#> [25] 1.273125 1.754750 1.147150 3.526900 1.740175 1.293375

Dawid-Sebastiani Score (DSS) @@ -211,14 +211,14 @@

Dawid-Sebastiani Score (DSS)
-observed <- rpois(30, lambda = 1:30)
-predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
-dss_sample(observed, predicted)
-#>  [1]  0.8444483  0.5596322 16.5535366  2.4938895  2.4357968  2.6493248
-#>  [7]  2.0738007  3.3996446  2.5559438  2.4644238  2.6801078  3.8685390
-#> [13]  2.6291814  2.6575992  2.5963740  3.7304259  2.7806829  3.6480289
-#> [19]  3.1914089  3.7919271  3.1913286  3.0987188  3.1796194  3.5042409
-#> [25]  3.8410857  4.1611087  3.3980257  4.1634345  3.9703019  3.8689574

+true_values <- rpois(30, lambda = 1:30) +predictions <- replicate(200, rpois(n = 30, lambda = 1:30)) +dss_sample(true_values, predictions) +#> [1] -0.1165337 1.2663719 1.3914896 1.6085479 1.6563335 3.1371559 +#> [7] 2.6572300 3.1818077 4.1452314 2.3744456 3.2213688 10.4516198 +#> [13] 2.4668204 5.3819668 8.5125715 3.9584451 2.8458896 2.9625067 +#> [19] 3.2815188 3.4173639 4.5575574 2.9095986 3.2911797 4.7701829 +#> [25] 3.0215461 5.6653143 4.2229032 8.9222126 6.9116543 3.8059803

Log Score @@ -231,36 +231,36 @@

Log Score
-observed <- rnorm(30, mean = 1:30)
-predicted <- replicate(200, rnorm(n = 30, mean = 1:30))
-logs_sample(observed, predicted)
-#>  [1] 1.3639180 1.1721622 0.9837552 1.1223984 1.2141230 1.1787707 0.9073846
-#>  [8] 1.9390409 1.2006537 1.5294916 0.9598881 1.9307719 1.1043973 1.0898149
-#> [15] 1.0772859 1.1025971 0.9462241 0.8310544 3.3920656 0.9183323 1.1210145
-#> [22] 1.1412519 1.3032000 2.9945874 1.2448300 1.1788962 1.1406896 1.2570678
-#> [29] 1.2374300 1.0551493

+true_values <- rnorm(30, mean = 1:30) +predictions <- replicate(200, rnorm(n = 30, mean = 1:30)) +logs_sample(true_values, predictions) +#> [1] 2.4832258 0.9598636 1.4306410 1.2242415 0.8851433 1.9428980 1.4357992 +#> [8] 3.2569086 1.2981616 1.4420158 1.7005497 1.2201677 2.1093755 1.2559861 +#> [15] 2.0107458 1.0006000 0.9992852 1.0830716 0.9463319 1.1916367 1.8491261 +#> [22] 2.7612405 0.9993017 1.0866942 1.1219392 1.0157949 4.3483059 1.5328976 +#> [29] 0.9696125 1.2730333

Brier Score

The Brier score is a proper score rule that assesses the accuracy of probabilistic binary predictions. The outcomes can be either 0 or 1, the -predictions must be a probability that the observed outcome will be -1.

+predictions must be a probability that the true outcome will be 1.

The Brier Score is then computed as the mean squared error between -the probabilistic prediction and the observed outcome.

+the probabilistic prediction and the true outcome.

\[\text{Brier_Score} = \frac{1}{N} \sum_{t = 1}^{n} (\text{prediction}_t - \text{outcome}_t)^2\]

-observed <- factor(sample(c(0, 1), size = 30, replace = TRUE))
-predicted <- runif(n = 30, min = 0, max = 1)
+true_values <- sample(c(0, 1), size = 30, replace = TRUE)
+predictions <- runif(n = 30, min = 0, max = 1)
 
-brier_score(observed, predicted)
-#>  [1] 0.112259247 0.564562618 0.222293787 0.021831261 0.790523726 0.440207593
-#>  [7] 0.317289909 0.010112601 0.063597143 0.557275438 0.867665126 0.288636782
-#> [13] 0.723326701 0.043212427 0.003723609 0.676499784 0.383250903 0.483864329
-#> [19] 0.868022259 0.016770692 0.207787622 0.485208168 0.965358526 0.442452645
-#> [25] 0.065318462 0.104556347 0.861472188 0.334785513 0.042639655 0.795482392
+brier_score(true_values, predictions) +#> [1] 0.0835547998 0.0034896980 0.6845269436 0.0071864763 0.0696040573 +#> [6] 0.7676418276 0.0006379362 0.7518769143 0.4812330379 0.0054759065 +#> [11] 0.1049385648 0.0259172107 0.1036774654 0.1103308393 0.7986939431 +#> [16] 0.0012878684 0.3327300596 0.0288541795 0.4666653522 0.0141451157 +#> [21] 0.7496970191 0.4247273749 0.0005254546 0.3743598068 0.9719426446 +#> [26] 0.8120250397 0.4121635979 0.4440633809 0.5460942947 0.0855679862

Interval Score

@@ -270,10 +270,10 @@

Interval ScoreThe score is computed as

\[ \text{score} = (\text{upper} - \text{lower}) + \\ -\frac{2}{\alpha} \cdot (\text{lower} - \text{observed}) \cdot -1(\text{observed} < \text{lower}) + \\ -\frac{2}{\alpha} \cdot (\text{observed} - \text{upper}) \cdot -1(\text{observed} > \text{upper})\]

+\frac{2}{\alpha} \cdot (\text{lower} - \text{true_value}) \cdot +1(\text{true_values} < \text{lower}) + \\ +\frac{2}{\alpha} \cdot (\text{true_value} - \text{upper}) \cdot +1(\text{true_value} > \text{upper})\]

where \(1()\) is the indicator function and \(\alpha\) is the decimal value that indicates how much is outside the prediction interval. To @@ -288,23 +288,23 @@

Interval Score 
-observed <- rnorm(30, mean = 1:30)
+true_values <- rnorm(30, mean = 1:30)
 interval_range <- 90
 alpha <- (100 - interval_range) / 100
 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30))
-upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40))
+upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 1:30))
 
 interval_score(
-  observed = observed,
+  true_values = true_values,
   lower = lower,
   upper = upper,
   interval_range = interval_range
 )
-#>  [1] 0.6670564 0.7459922 0.5539554 1.2462373 0.6698641 0.7588718 0.6326608
-#>  [8] 0.7020316 0.7733863 0.8357305 0.5821739 0.7204689 0.5707887 0.6594888
-#> [15] 0.7184740 0.7456016 1.8944255 0.6496035 0.5651959 0.6631289 0.6886410
-#> [22] 0.6441808 0.7138403 0.6287681 0.5477585 0.6669178 0.6891055 0.6213826
-#> [29] 0.6414575 0.9052694
+#> [1] 0.19791808 0.76229272 0.14294468 0.30458137 0.19208544 0.18084811 +#> [7] 0.10865077 0.15457993 0.14103317 0.15267216 0.19323575 0.34445089 +#> [13] 0.13789245 0.54415421 0.20897700 0.22212496 0.22952582 0.23314254 +#> [19] 0.11966374 0.21553324 0.03889604 0.06743880 0.23498491 0.26259314 +#> [25] 0.99429294 0.13451870 0.18773537 2.33687380 0.16288762 0.18674251