From a92b2a2699c084e7c2af3017fd369e8de7a366b3 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 14:55:22 -0600 Subject: [PATCH 01/23] Create first draft of `get_coverage()` --- NAMESPACE | 2 +- R/add_coverage.R | 47 ++++++++++++++---------- man/{add_coverage.Rd => get_coverage.Rd} | 13 ++++--- 3 files changed, 36 insertions(+), 26 deletions(-) rename man/{add_coverage.Rd => get_coverage.Rd} (89%) diff --git a/NAMESPACE b/NAMESPACE index a5147fd66..8df5afe85 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -22,7 +22,6 @@ S3method(validate_forecast,forecast_binary) S3method(validate_forecast,forecast_point) S3method(validate_forecast,forecast_quantile) S3method(validate_forecast,forecast_sample) -export(add_coverage) export(add_pairwise_comparison) export(ae_median_quantile) export(ae_median_sample) @@ -35,6 +34,7 @@ export(correlation) export(crps_sample) export(dispersion) export(dss_sample) +export(get_coverage) export(get_duplicate_forecasts) export(get_forecast_counts) export(get_forecast_type) diff --git a/R/add_coverage.R b/R/add_coverage.R index 405ca3a06..564e693aa 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -1,6 +1,6 @@ -#' @title Add Coverage Values to Quantile-Based Forecasts +#' @title Get Quantile And Interval Coverage Values For Quantile-Based Forecasts #' -#' @description Adds interval coverage of central prediction intervals, +#' @description Compute interval coverage of central prediction intervals, #' quantile coverage for predictive quantiles, as well as the deviation between #' desired and actual coverage to a data.table. Forecasts should be in a #' quantile format (following the input requirements of `score()`). @@ -42,44 +42,53 @@ #' "interval_coverage_deviation", "quantile_coverage", #' "quantile_coverage_deviation" added. #' @importFrom data.table setcolorder +#' @importFrom checkmate assert_subset #' @examples #' library(magrittr) # pipe operator #' example_quantile %>% -#' add_coverage() +#' as_forecast() %>% +#' get_coverage(by = "model") #' @export #' @keywords scoring #' @export -add_coverage <- function(data) { - stored_attributes <- get_scoringutils_attributes(data) - data <- as_forecast(data) - forecast_unit <- get_forecast_unit(data) - data_cols <- colnames(data) # store so we can reset column order later +get_coverage <- function(data, by = get_forecast_unit(data)) { + # input checks --------------------------------------------------------------- + data <- as_forecast(na.omit(data), forecast_type = "quantile") + assert_subset(by, names(data)) + # convert to wide interval format and compute interval coverage -------------- interval_data <- quantile_to_interval(data, format = "wide") interval_data[, - interval_coverage := (observed <= upper) & (observed >= lower) + interval_coverage := (observed <= upper) & (observed >= lower) ][, c("lower", "upper", "observed") := NULL] + interval_data[, interval_coverage_deviation := + interval_coverage - interval_range / 100] + # merge interval range data with original data ------------------------------- + # preparations data[, interval_range := get_range_from_quantile(quantile_level)] + data_cols <- colnames(data) # store so we can reset column order later + forecast_unit <- get_forecast_unit(data) data <- merge(data, interval_data, by = unique(c(forecast_unit, "interval_range"))) - data[, interval_coverage_deviation := - interval_coverage - interval_range / 100] + + # compute quantile coverage and deviation ------------------------------------ data[, quantile_coverage := observed <= predicted] data[, quantile_coverage_deviation := quantile_coverage - quantile_level] + # summarise coverage values according to `by` and cleanup -------------------- # reset column order new_metrics <- c("interval_coverage", "interval_coverage_deviation", "quantile_coverage", "quantile_coverage_deviation") setcolorder(data, unique(c(data_cols, "interval_range", new_metrics))) - - # add coverage "metrics" to list of stored metrics - # this makes it possible to use `summarise_scores()` later on - stored_attributes[["score_names"]] <- c( - stored_attributes[["score_names"]], - new_metrics - ) - data <- assign_attributes(data, stored_attributes) + # remove forecast class and convert to regular data.table + data <- as.data.table(data) + by <- unique(c(by, "quantile_level", "interval_range")) + # summarise + data <- data[, lapply(.SD, mean), + by = by, + .SDcols = new_metrics + ] return(data[]) } diff --git a/man/add_coverage.Rd b/man/get_coverage.Rd similarity index 89% rename from man/add_coverage.Rd rename to man/get_coverage.Rd index addd1f72d..8557054ba 100644 --- a/man/add_coverage.Rd +++ b/man/get_coverage.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/add_coverage.R -\name{add_coverage} -\alias{add_coverage} -\title{Add Coverage Values to Quantile-Based Forecasts} +\name{get_coverage} +\alias{get_coverage} +\title{Get Quantile And Interval Coverage Values For Quantile-Based Forecasts} \usage{ -add_coverage(data) +get_coverage(data, by = get_forecast_unit(data)) } \arguments{ \item{data}{A data.frame or data.table with predicted and observed values.} @@ -15,7 +15,7 @@ a data.table with the input and columns "interval_coverage", "quantile_coverage_deviation" added. } \description{ -Adds interval coverage of central prediction intervals, +Compute interval coverage of central prediction intervals, quantile coverage for predictive quantiles, as well as the deviation between desired and actual coverage to a data.table. Forecasts should be in a quantile format (following the input requirements of \code{score()}). @@ -55,6 +55,7 @@ coverage is 80\%, the coverage deviation is -0.1. \examples{ library(magrittr) # pipe operator example_quantile \%>\% - add_coverage() + as_forecast() \%>\% + get_coverage(by = "model") } \keyword{scoring} From 1303d716a2edec57c771832e04c66a8174dfee51 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 19:22:49 -0600 Subject: [PATCH 02/23] Remove a few mentions of `add_coverage()` and replace by `get_coverage()` --- NEWS.md | 2 +- R/add_coverage.R | 14 ++++++++------ R/get_-functions.R | 4 ++-- R/plot.R | 10 ++++------ R/score.R | 4 +--- README.Rmd | 3 +-- man/get_coverage.Rd | 14 ++++++++------ man/get_score_names.Rd | 4 ++-- man/plot_interval_coverage.Rd | 5 ++--- man/plot_quantile_coverage.Rd | 5 ++--- 10 files changed, 31 insertions(+), 34 deletions(-) diff --git a/NEWS.md b/NEWS.md index 19714dc68..0a3adc1ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,7 +26,7 @@ The update introduces breaking changes. If you want to keep using the older vers - `check_forecasts()` was replaced by a different workflow. There now is a function, `as_forecast()`, that determines forecast type of the data, constructs a forecasting object and validates it using the function `validate_forecast()` (a generic that dispatches the correct method based on the forecast type). Objects of class `forecast_binary`, `forecast_point`, `forecast_sample` and `forecast_quantile` have print methods that fulfill the functionality of `check_forecasts()`. - Users can test whether an object is of class `forecast_*()` using the function `is_forecast()`. Users can also test for a specific `forecast_*` class using the appropriate `is_forecast.forecast_*` method. For example, to check whether an object is of class `forecast_quantile`, you would use you would use `scoringutils:::is_forecast.forecast_quantile()`. - The functionality for computing pairwise comparisons was now split from `summarise_scores()`. Instead of doing pairwise comparisons as part of summarising scores, a new function, `add_pairwise_comparison()`, was introduced that takes summarised scores as an input and adds columns with relative skil scores and scaled relative skill scores. -- `add_coverage()` was reworked completely. It's new purpose is now to add coverage information to the raw forecast data (essentially fulfilling some of the functionality that was previously covered by `score_quantile()`) +- `add_coverage()` was replaced by a new function, `get_coverage()`. This function comes with an updated workflow where coverage values are computed directly based on the original data and can then be visualised using `plot_interval_coverage()` or `plot_quantile_coverage()`. An example worfklow would be `example_quantile |> as_forecast() |> get_coverage(by = "model") |> plot_interval_coverage()`. - Support for the interval format was mostly dropped (see PR #525 by @nikosbosse and reviewed by @seabbs) - The function `bias_range()` was removed (users should now use `bias_quantile()` instead) - The function `interval_score()` was made an internal function rather than being exported to users. We recommend using `wis()` instead. diff --git a/R/add_coverage.R b/R/add_coverage.R index 564e693aa..b56e3b9cd 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -14,12 +14,14 @@ #' central prediction interval is the interval between the 0.25 and 0.75 #' quantiles of the predictive distribution. #' -#' The function `add_coverage()` computes the coverage per central prediction -#' interval, so the interval coverage will always be either `TRUE` -#' (observed value falls within the interval) or `FALSE` (observed value falls -#' outside the interval). You can summarise the interval coverage values to get -#' the proportion of observations that fall within the central prediction -#' intervals. +#' The function `get_coverage()` computes the coverage per central prediction +#' interval. This means that if you set `by` to the unit of a single forecast, +#' interval coverage will always be either `TRUE` +#' (observed value falls within the interval) or `FALSE` (observed value falls +#' outside the interval) and analogously for quantile coverage. +#' Coverage values become meaningful by summarising them across different +#' dimensions, as specified in the `by` argument (thereby returning the +#' proportion of values covered by all prediction intervals/quantiles). #' #' **Quantile coverage** #' diff --git a/R/get_-functions.R b/R/get_-functions.R index c2ec9a223..f5677403e 100644 --- a/R/get_-functions.R +++ b/R/get_-functions.R @@ -125,8 +125,8 @@ get_type <- function(x) { #' @title Get Names Of The Scoring Rules That Were Used For Scoring #' @description -#' When applying a scoring rule, (for example through [score()] or -#' [add_coverage()], the names of the scoring rules become column names of the +#' When applying a scoring rule via [score()], the names of the scoring rules +#' become column names of the #' resulting data.table. In addition, an attribute `score_names` will be #' added to the output, holding the names of the scores as a vector. #' This is done so that a function like [get_forecast_unit()] can still diff --git a/R/plot.R b/R/plot.R index 31b410b76..7ff277b49 100644 --- a/R/plot.R +++ b/R/plot.R @@ -521,9 +521,8 @@ make_na <- make_NA #' \dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } -#' data_coverage <- add_coverage(example_quantile) -#' summarised <- summarise_scores(data_coverage, by = c("model", "interval_range")) -#' plot_interval_coverage(summarised) +#' coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +#' plot_interval_coverage(coverage) plot_interval_coverage <- function(coverage, colour = "model") { ## overall model calibration - empirical interval coverage @@ -573,9 +572,8 @@ plot_interval_coverage <- function(coverage, #' @importFrom data.table dcast #' @export #' @examples -#' data_coverage <- add_coverage(example_quantile) -#' summarised <- summarise_scores(data_coverage, by = c("model", "quantile_level")) -#' plot_quantile_coverage(summarised) +#' coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +#' plot_quantile_coverage(coverage) plot_quantile_coverage <- function(coverage, colour = "model") { diff --git a/R/score.R b/R/score.R index 3ef976e02..b65bcd835 100644 --- a/R/score.R +++ b/R/score.R @@ -194,9 +194,7 @@ score.forecast_quantile <- function(data, metrics = rules_quantile(), ...) { }) scores <- rbindlist(split_result) - # this can have existing scores, e.g. from `add_coverage()` - existing_scores <- get_score_names(data) - scores <- as_scores(scores, score_names = c(existing_scores, names(metrics))) + scores <- as_scores(scores, score_names = names(metrics)) return(scores[]) } diff --git a/README.Rmd b/README.Rmd index d22550647..4654fb0f7 100644 --- a/README.Rmd +++ b/README.Rmd @@ -121,14 +121,13 @@ example_quantile %>% ### Scoring forecasts Forecasts can be easily and quickly scored using the `score()` function. `score()` automatically tries to determine the `forecast_unit`, i.e. the set of columns that uniquely defines a single forecast, by taking all column names of the data into account. However, it is recommended to set the forecast unit manually by specifying the "forecast_unit" argument in `as_forecast()` as this may help to avoid errors. This will drop all columns that are neither part of the forecast unit nor part of the columns internally used by `scoringutils`. The function `as_forecast()` processes and validates the inputs. -`score()` returns unsummarised scores, which in most cases is not what the user wants. Here we make use of additional functions from `scoringutils` to add empirical coverage-levels (`add_coverage()`), and scores relative to a baseline model (here chosen to be the EuroCOVIDhub-ensemble model). See the getting started vignette for more details. Finally we summarise these scores by model and target type. +`score()` returns unsummarised scores, which in most cases is not what the user wants. Here we make use of an additional function from `scoringutils` to add scores relative to a baseline model (here chosen to be the EuroCOVIDhub-ensemble model). See the getting started vignette for more details. Finally we summarise these scores by model and target type. ```{r score-example} example_quantile %>% as_forecast(forecast_unit = c( "location", "target_end_date", "target_type", "horizon", "model" )) %>% - add_coverage() %>% score() %>% add_pairwise_comparison( by = c("model", "target_type"), diff --git a/man/get_coverage.Rd b/man/get_coverage.Rd index 8557054ba..6ebd266c3 100644 --- a/man/get_coverage.Rd +++ b/man/get_coverage.Rd @@ -29,12 +29,14 @@ by two quantiles that denote the lower and upper bound. For example, the 50\% central prediction interval is the interval between the 0.25 and 0.75 quantiles of the predictive distribution. -The function \code{add_coverage()} computes the coverage per central prediction -interval, so the interval coverage will always be either \code{TRUE} -(observed value falls within the interval) or \code{FALSE} (observed value falls -outside the interval). You can summarise the interval coverage values to get -the proportion of observations that fall within the central prediction -intervals. +The function \code{get_coverage()} computes the coverage per central prediction +interval. This means that if you set \code{by} to the unit of a single forecast, +interval coverage will always be either \code{TRUE} +(observed value falls within the interval) or \code{FALSE} (observed value falls +outside the interval) and analogously for quantile coverage. +Coverage values become meaningful by summarising them across different +dimensions, as specified in the \code{by} argument (thereby returning the +proportion of values covered by all prediction intervals/quantiles). \strong{Quantile coverage} diff --git a/man/get_score_names.Rd b/man/get_score_names.Rd index c2b3baf44..23c8700d9 100644 --- a/man/get_score_names.Rd +++ b/man/get_score_names.Rd @@ -17,8 +17,8 @@ Character vector with the names of the scoring rules that were used for scoring or \code{NULL} if no scores were computed previously. } \description{ -When applying a scoring rule, (for example through \code{\link[=score]{score()}} or -\code{\link[=add_coverage]{add_coverage()}}, the names of the scoring rules become column names of the +When applying a scoring rule via \code{\link[=score]{score()}}, the names of the scoring rules +become column names of the resulting data.table. In addition, an attribute \code{score_names} will be added to the output, holding the names of the scores as a vector. This is done so that a function like \code{\link[=get_forecast_unit]{get_forecast_unit()}} can still diff --git a/man/plot_interval_coverage.Rd b/man/plot_interval_coverage.Rd index 90d1e6499..271fe1665 100644 --- a/man/plot_interval_coverage.Rd +++ b/man/plot_interval_coverage.Rd @@ -23,7 +23,6 @@ Plot interval coverage \dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN } -data_coverage <- add_coverage(example_quantile) -summarised <- summarise_scores(data_coverage, by = c("model", "interval_range")) -plot_interval_coverage(summarised) +coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +plot_interval_coverage(coverage) } diff --git a/man/plot_quantile_coverage.Rd b/man/plot_quantile_coverage.Rd index bdec00d94..cdbda701f 100644 --- a/man/plot_quantile_coverage.Rd +++ b/man/plot_quantile_coverage.Rd @@ -20,7 +20,6 @@ ggplot object with a plot of interval coverage Plot quantile coverage } \examples{ -data_coverage <- add_coverage(example_quantile) -summarised <- summarise_scores(data_coverage, by = c("model", "quantile_level")) -plot_quantile_coverage(summarised) +coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +plot_quantile_coverage(coverage) } From 1296b435982bc00161355c225895cd94ad0749e0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 29 Feb 2024 01:25:11 +0000 Subject: [PATCH 03/23] Automatic readme update [ci skip] --- README.md | 42 +++++++++++++----------------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 66bf5c99c..7ae5b47a8 100644 --- a/README.md +++ b/README.md @@ -140,18 +140,16 @@ columns that are neither part of the forecast unit nor part of the columns internally used by `scoringutils`. The function `as_forecast()` processes and validates the inputs. `score()` returns unsummarised scores, which in most cases is not what the user wants. Here we make use -of additional functions from `scoringutils` to add empirical -coverage-levels (`add_coverage()`), and scores relative to a baseline -model (here chosen to be the EuroCOVIDhub-ensemble model). See the -getting started vignette for more details. Finally we summarise these -scores by model and target type. +of an additional function from `scoringutils` to add scores relative to +a baseline model (here chosen to be the EuroCOVIDhub-ensemble model). +See the getting started vignette for more details. Finally we summarise +these scores by model and target type. ``` r example_quantile %>% as_forecast(forecast_unit = c( "location", "target_end_date", "target_type", "horizon", "model" )) %>% - add_coverage() %>% score() %>% add_pairwise_comparison( by = c("model", "target_type"), @@ -167,31 +165,17 @@ example_quantile %>% kable() #> Some rows containing NA values may be removed. This is fine if not unexpected. #> Some rows containing NA values may be removed. This is fine if not unexpected. -#> Some rows containing NA values may be removed. This is fine if not unexpected. -#> Warning in get_score_names(scores, error = TRUE): The following scores have -#> been previously computed, but are no longer column names of the data: -#> `interval_coverage, quantile_coverage, quantile_coverage_deviation`. See -#> `?get_score_names` for further information. - -#> Warning in get_score_names(scores, error = TRUE): The following scores have -#> been previously computed, but are no longer column names of the data: -#> `interval_coverage, quantile_coverage, quantile_coverage_deviation`. See -#> `?get_score_names` for further information. -#> Warning in get_score_names(scores): The following scores have been previously -#> computed, but are no longer column names of the data: `interval_coverage, -#> quantile_coverage, quantile_coverage_deviation`. See `?get_score_names` for -#> further information. ``` -| model | target_type | wis | overprediction | underprediction | dispersion | bias | interval_coverage_50 | interval_coverage_90 | interval_coverage_deviation | ae_median | wis_relative_skill | wis_scaled_relative_skill | -|:----------------------|:------------|------:|---------------:|----------------:|-----------:|--------:|---------------------:|---------------------:|----------------------------:|----------:|-------------------:|--------------------------:| -| EuroCOVIDhub-baseline | Cases | 28000 | 14000.0 | 10000.0 | 4100 | 0.0980 | 0.33 | 0.82 | -0.120 | 38000 | 1.30 | 1.6 | -| EuroCOVIDhub-baseline | Deaths | 160 | 66.0 | 2.1 | 91 | 0.3400 | 0.66 | 1.00 | 0.120 | 230 | 2.30 | 3.8 | -| EuroCOVIDhub-ensemble | Cases | 18000 | 10000.0 | 4200.0 | 3700 | -0.0560 | 0.39 | 0.80 | -0.100 | 24000 | 0.82 | 1.0 | -| EuroCOVIDhub-ensemble | Deaths | 41 | 7.1 | 4.1 | 30 | 0.0730 | 0.88 | 1.00 | 0.200 | 53 | 0.60 | 1.0 | -| UMass-MechBayes | Deaths | 53 | 9.0 | 17.0 | 27 | -0.0220 | 0.46 | 0.88 | -0.025 | 78 | 0.75 | 1.3 | -| epiforecasts-EpiNow2 | Cases | 21000 | 12000.0 | 3300.0 | 5700 | -0.0790 | 0.47 | 0.79 | -0.070 | 28000 | 0.95 | 1.2 | -| epiforecasts-EpiNow2 | Deaths | 67 | 19.0 | 16.0 | 32 | -0.0051 | 0.42 | 0.91 | -0.045 | 100 | 0.98 | 1.6 | +| model | wis | overprediction | underprediction | dispersion | bias | interval_coverage_50 | interval_coverage_90 | interval_coverage_deviation | ae_median | wis_relative_skill | wis_scaled_relative_skill | +|:----------------------|------:|---------------:|----------------:|-----------:|--------:|---------------------:|---------------------:|----------------------------:|----------:|-------------------:|--------------------------:| +| EuroCOVIDhub-baseline | 28000 | 14000.0 | 10000.0 | 4100 | 0.0980 | 0.33 | 0.82 | -0.120 | 38000 | 1.30 | 1.6 | +| EuroCOVIDhub-baseline | 160 | 66.0 | 2.1 | 91 | 0.3400 | 0.66 | 1.00 | 0.120 | 230 | 2.30 | 3.8 | +| EuroCOVIDhub-ensemble | 18000 | 10000.0 | 4200.0 | 3700 | -0.0560 | 0.39 | 0.80 | -0.100 | 24000 | 0.82 | 1.0 | +| EuroCOVIDhub-ensemble | 41 | 7.1 | 4.1 | 30 | 0.0730 | 0.88 | 1.00 | 0.200 | 53 | 0.60 | 1.0 | +| UMass-MechBayes | 53 | 9.0 | 17.0 | 27 | -0.0220 | 0.46 | 0.88 | -0.025 | 78 | 0.75 | 1.3 | +| epiforecasts-EpiNow2 | 21000 | 12000.0 | 3300.0 | 5700 | -0.0790 | 0.47 | 0.79 | -0.070 | 28000 | 0.95 | 1.2 | +| epiforecasts-EpiNow2 | 67 | 19.0 | 16.0 | 32 | -0.0051 | 0.42 | 0.91 | -0.045 | 100 | 0.98 | 1.6 | `scoringutils` contains additional functionality to transform forecasts, to summarise scores at different levels, to visualise them, and to From 1461e8853a1e47710d4e3f8fa697bb5d2088f54d Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 21:10:16 -0600 Subject: [PATCH 04/23] Replace more instances of `add_coverage()` --- README.md | 2 +- tests/testthat/test-add_coverage.R | 27 +++++++++----------- tests/testthat/test-plot_interval_coverage.R | 3 +-- tests/testthat/test-plot_quantile_coverage.R | 3 +-- vignettes/scoringutils.Rmd | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7ae5b47a8..289dd4366 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ example_quantile %>% #> underprediction dispersion bias interval_coverage_50 #> #> 1: 4237.177310 3663.52458 -0.05640625 0.3906250 -#> 2: 10284.972826 4102.50094 0.09726562 0.3281250 +#> 2: 10284.972826 4102.50094 0.09726563 0.3281250 #> 3: 3260.355639 5664.37795 -0.07890625 0.4687500 #> 4: 4.103261 30.18099 0.07265625 0.8750000 #> 5: 2.098505 91.40625 0.33906250 0.6640625 diff --git a/tests/testthat/test-add_coverage.R b/tests/testthat/test-add_coverage.R index e615253b6..e8a17ed3d 100644 --- a/tests/testthat/test-add_coverage.R +++ b/tests/testthat/test-add_coverage.R @@ -1,24 +1,21 @@ ex_coverage <- example_quantile[model == "EuroCOVIDhub-ensemble"] -test_that("add_coverage() works as expected", { - expect_message( - cov <- add_coverage(example_quantile), - "Some rows containing NA values may be removed." - ) +test_that("get_coverage() works as expected", { + cov <- get_coverage(example_quantile, by = get_forecast_unit(example_quantile)) - required_names <- c( - "interval_range", "interval_coverage", "interval_coverage_deviation", - "quantile_coverage", "quantile_coverage_deviation" + expect_equal( + sort(colnames(cov)), + sort(c(get_forecast_unit(example_quantile), c( + "interval_range", "quantile_level", "interval_coverage", "interval_coverage_deviation", + "quantile_coverage", "quantile_coverage_deviation" + ))) ) - expect_equal(colnames(cov), c(colnames(example_quantile), required_names)) - - expect_equal(nrow(cov), nrow(example_quantile)) - # check that + expect_equal(nrow(cov), nrow(na.omit(example_quantile))) }) -test_that("add_coverage() outputs an object of class forecast_*", { +test_that("get_coverage() outputs an object of class c('data.table', 'data.frame'", { ex <- as_forecast(na.omit(example_quantile)) - cov <- add_coverage(ex) - expect_s3_class(cov, "forecast_quantile") + cov <- get_coverage(ex) + expect_s3_class(cov, c("data.table", "data.frame"), exact = TRUE) }) diff --git a/tests/testthat/test-plot_interval_coverage.R b/tests/testthat/test-plot_interval_coverage.R index fcf08c7aa..3c65fc538 100644 --- a/tests/testthat/test-plot_interval_coverage.R +++ b/tests/testthat/test-plot_interval_coverage.R @@ -1,6 +1,5 @@ test_that("plot_interval_coverage() works as expected", { - coverage <- add_coverage(na.omit(example_quantile)) %>% - summarise_scores(by = c("model", "interval_range")) + coverage <- get_coverage(example_quantile, by = c("model")) p <- plot_interval_coverage(coverage) expect_s3_class(p, "ggplot") skip_on_cran() diff --git a/tests/testthat/test-plot_quantile_coverage.R b/tests/testthat/test-plot_quantile_coverage.R index 133d2e35d..f53d2c22f 100644 --- a/tests/testthat/test-plot_quantile_coverage.R +++ b/tests/testthat/test-plot_quantile_coverage.R @@ -1,6 +1,5 @@ test_that("plot_quantile_coverage() works as expected", { - coverage <- add_coverage(na.omit(example_quantile)) %>% - summarise_scores(by = c("model", "quantile_level")) + coverage <- get_coverage(example_quantile, by = c("model", "quantile_level")) p <- plot_quantile_coverage(coverage) expect_s3_class(p, "ggplot") diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index 257540276..356aba5b4 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -188,7 +188,7 @@ suppressMessages(score(example_point)) %>% ### Adding empirical coverage -For quantile-based forecasts we are often interested in specific coverage-levels, for example, what percentage of true values fell between all 50% or the 90% prediction intervals. We can add this information using the function `add_coverage()`. This function also requires a `by` argument which defines the level of grouping for which the percentage of true values covered by certain prediction intervals is computed. +For quantile-based forecasts we are often interested in specific coverage-levels, for example, what percentage of true values fell between all 50% or the 90% prediction intervals. We can add this information using the function `get_coverage()`. This function also requires a `by` argument which defines the level of grouping for which the percentage of true values covered by certain prediction intervals is computed. ```{r} score(example_quantile) %>% From 341846d5a77ab5c7ac941918b86268893c953a25 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 21:10:34 -0600 Subject: [PATCH 05/23] delete test related to printing scores --- tests/testthat/test-utils.R | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 7975d6421..82b9fbde4 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -105,18 +105,6 @@ test_that("print() works on forecast_* objects", { output_test <- capture.output(print(data.table(dat))) expect_contains(output_original, output_test) } - - # Check Score columns are printed - dat <- example_quantile %>% - na.omit %>% - set_forecast_unit(c("location", "target_end_date", - "target_type", "horizon", "model")) %>% - as_forecast() %>% - add_coverage() - - expect_output(print(dat), "Score columns") - score_cols <- get_score_names(dat) - expect_output(print(dat), pattern = paste(score_cols, collapse = " ")) }) test_that("print methods fail gracefully", { From 205d3069c8a8f1060cee6fc5bd476037c01bb8d9 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 21:10:52 -0600 Subject: [PATCH 06/23] Update `plot_interval_range()` to handle output of `get_coverage()` --- R/plot.R | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/R/plot.R b/R/plot.R index 7ff277b49..47dca605e 100644 --- a/R/plot.R +++ b/R/plot.R @@ -525,6 +525,13 @@ make_na <- make_NA #' plot_interval_coverage(coverage) plot_interval_coverage <- function(coverage, colour = "model") { + # in case quantile columns are present, remove them and then take unique + # values. This doesn't visually affect the plot, but prevents lines from being + # drawn twice. + del <- c("quantile_level", "quantile_coverage", "quantile_coverage_deviation") + suppressWarnings(coverage[, eval(del) := NULL]) + coverage <- unique(coverage) + ## overall model calibration - empirical interval coverage p1 <- ggplot(coverage, aes( x = interval_range, From 42a6c870bfe3baa349355fffff91fa4f21cb2ed4 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 21:47:31 -0600 Subject: [PATCH 07/23] Update default arg for `by` --- R/add_coverage.R | 15 +++++++++++---- man/get_coverage.Rd | 12 ++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/R/add_coverage.R b/R/add_coverage.R index b56e3b9cd..f262abbf3 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -9,7 +9,7 @@ #' #' Interval coverage for a given interval range is defined as the proportion of #' observations that fall within the corresponding central prediction intervals. -#' Central prediction intervals are symmetric around the median and and formed +#' Central prediction intervals are symmetric around the median and formed #' by two quantiles that denote the lower and upper bound. For example, the 50% #' central prediction interval is the interval between the 0.25 and 0.75 #' quantiles of the predictive distribution. @@ -40,9 +40,12 @@ #' coverage is 80%, the coverage deviation is -0.1. #' #' @inheritParams score -#' @return a data.table with the input and columns "interval_coverage", +#' @param by character vector that denotes the level of grouping for which the +#' coverage values should be computed. By default (`"model"`), one coverage +#' value per model will be returned. +#' @return a data.table with columns "interval_coverage", #' "interval_coverage_deviation", "quantile_coverage", -#' "quantile_coverage_deviation" added. +#' "quantile_coverage_deviation" and the columns specified in `by`. #' @importFrom data.table setcolorder #' @importFrom checkmate assert_subset #' @examples @@ -53,9 +56,13 @@ #' @export #' @keywords scoring #' @export -get_coverage <- function(data, by = get_forecast_unit(data)) { +get_coverage <- function(data, by = "model") { # input checks --------------------------------------------------------------- data <- as_forecast(na.omit(data), forecast_type = "quantile") + + # remove "quantile_level" and "interval_range" from `by` if present, as these + # are included anyway + by <- setdiff(by, c("quantile_level", "interval_range")) assert_subset(by, names(data)) # convert to wide interval format and compute interval coverage -------------- diff --git a/man/get_coverage.Rd b/man/get_coverage.Rd index 6ebd266c3..eea37bf44 100644 --- a/man/get_coverage.Rd +++ b/man/get_coverage.Rd @@ -4,15 +4,19 @@ \alias{get_coverage} \title{Get Quantile And Interval Coverage Values For Quantile-Based Forecasts} \usage{ -get_coverage(data, by = get_forecast_unit(data)) +get_coverage(data, by = "model") } \arguments{ \item{data}{A data.frame or data.table with predicted and observed values.} + +\item{by}{character vector that denotes the level of grouping for which the +coverage values should be computed. By default (\code{"model"}), one coverage +value per model will be returned.} } \value{ -a data.table with the input and columns "interval_coverage", +a data.table with columns "interval_coverage", "interval_coverage_deviation", "quantile_coverage", -"quantile_coverage_deviation" added. +"quantile_coverage_deviation" and the columns specified in \code{by}. } \description{ Compute interval coverage of central prediction intervals, @@ -24,7 +28,7 @@ quantile format (following the input requirements of \code{score()}). Interval coverage for a given interval range is defined as the proportion of observations that fall within the corresponding central prediction intervals. -Central prediction intervals are symmetric around the median and and formed +Central prediction intervals are symmetric around the median and formed by two quantiles that denote the lower and upper bound. For example, the 50\% central prediction interval is the interval between the 0.25 and 0.75 quantiles of the predictive distribution. From 4ab96cd55d385446d30e157025b64c5d582826ac Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 21:47:52 -0600 Subject: [PATCH 08/23] Update example for `get_coverage()` --- R/plot.R | 2 +- man/plot_interval_coverage.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/plot.R b/R/plot.R index 47dca605e..a59db71f7 100644 --- a/R/plot.R +++ b/R/plot.R @@ -521,7 +521,7 @@ make_na <- make_NA #' \dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } -#' coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +#' coverage <- get_coverage(example_quantile, by = c("model")) #' plot_interval_coverage(coverage) plot_interval_coverage <- function(coverage, colour = "model") { diff --git a/man/plot_interval_coverage.Rd b/man/plot_interval_coverage.Rd index 271fe1665..09dce333a 100644 --- a/man/plot_interval_coverage.Rd +++ b/man/plot_interval_coverage.Rd @@ -23,6 +23,6 @@ Plot interval coverage \dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN } -coverage <- get_coverage(example_quantile, by = c("model", "interval_range")) +coverage <- get_coverage(example_quantile, by = c("model")) plot_interval_coverage(coverage) } From 43d6af3ae929867a63ed59cef25e48c767d00b0c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:02:06 -0600 Subject: [PATCH 09/23] Add tests to check behaviour of `get_coverage()` when prediction intervals are not symmetric --- tests/testthat/test-add_coverage.R | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/testthat/test-add_coverage.R b/tests/testthat/test-add_coverage.R index e8a17ed3d..fb177a767 100644 --- a/tests/testthat/test-add_coverage.R +++ b/tests/testthat/test-add_coverage.R @@ -19,3 +19,27 @@ test_that("get_coverage() outputs an object of class c('data.table', 'data.frame cov <- get_coverage(ex) expect_s3_class(cov, c("data.table", "data.frame"), exact = TRUE) }) + +test_that("get_coverage() can deal with non-symmetric prediction intervals", { + # the expected result is that `get_coverage()` just works. However, + # all interval coverages with missing values should just be `NA` + test <- data.table::copy(example_quantile) + test <- test[!quantile_level %in% c(0.2, 0.3, 0.5)] + + expect_no_condition(cov <- get_coverage(test)) + + prediction_intervals <- get_range_from_quantile(c(0.2, 0.3, 0.5)) + + missing <- cov[interval_range %in% prediction_intervals] + not_missing <- cov[!interval_range %in% prediction_intervals] + + expect_true(all(is.na(missing$interval_coverage))) + expect_false(any(is.na(not_missing))) + + # test for a version where values are not missing, but just `NA` + # since `get_coverage()` calls `na.omit`, the result should be the same. + test <- data.table::copy(example_quantile) + test <- test[quantile_level %in% c(0.2, 0.3, 0.5), predicted := NA] + cov2 <- get_coverage(test) + expect_equal(cov, cov2) +}) From 82a67608cc1e84298dfdfd992c429caa9210048a Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:07:01 -0600 Subject: [PATCH 10/23] fix linting issues --- R/add_coverage.R | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/R/add_coverage.R b/R/add_coverage.R index f262abbf3..7ee239e11 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -68,7 +68,7 @@ get_coverage <- function(data, by = "model") { # convert to wide interval format and compute interval coverage -------------- interval_data <- quantile_to_interval(data, format = "wide") interval_data[, - interval_coverage := (observed <= upper) & (observed >= lower) + interval_coverage := (observed <= upper) & (observed >= lower) ][, c("lower", "upper", "observed") := NULL] interval_data[, interval_coverage_deviation := interval_coverage - interval_range / 100] @@ -95,9 +95,6 @@ get_coverage <- function(data, by = "model") { data <- as.data.table(data) by <- unique(c(by, "quantile_level", "interval_range")) # summarise - data <- data[, lapply(.SD, mean), - by = by, - .SDcols = new_metrics - ] + data <- data[, lapply(.SD, mean), by = by, .SDcols = new_metrics] return(data[]) } From e0a541a105bd933303cb0f1e23276d4f5cfab37b Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:50:05 -0600 Subject: [PATCH 11/23] Delete function `assign_attributes()` --- R/utils.R | 19 ------------------- man/assign_attributes.Rd | 21 --------------------- 2 files changed, 40 deletions(-) delete mode 100644 man/assign_attributes.Rd diff --git a/R/utils.R b/R/utils.R index 983cebde2..d074fda3b 100644 --- a/R/utils.R +++ b/R/utils.R @@ -56,25 +56,6 @@ filter_function_args <- function(fun, args) { } -#' @title Assign attributes to an object from a named list -#' @description -#' Every list item will be made an attribute of the object. -#' @param object An object to assign attributes to -#' @param attribute_list A named list of attributes to assign to the object. -#' -#' @return The object with new attributes according to the contents of -#' `attribute_list` -#' @keywords internal -assign_attributes <- function(object, attribute_list) { - if (is.null(object)) { - return(NULL) - } - for (i in seq_along(attribute_list)) { - setattr(object, names(attribute_list)[i], attribute_list[[i]]) - } - return(object) -} - #' Strip attributes from an object #' @description This function removes all attributes from an object that are #' specified in the `attributes` argument. diff --git a/man/assign_attributes.Rd b/man/assign_attributes.Rd deleted file mode 100644 index f6dfdeadb..000000000 --- a/man/assign_attributes.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R -\name{assign_attributes} -\alias{assign_attributes} -\title{Assign attributes to an object from a named list} -\usage{ -assign_attributes(object, attribute_list) -} -\arguments{ -\item{object}{An object to assign attributes to} - -\item{attribute_list}{A named list of attributes to assign to the object.} -} -\value{ -The object with new attributes according to the contents of -\code{attribute_list} -} -\description{ -Every list item will be made an attribute of the object. -} -\keyword{internal} From cf112ba94ad6fc0117b5097e92e6a7dd4e07e3bb Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:54:15 -0600 Subject: [PATCH 12/23] Delete function `strip_attributes()` --- R/utils.R | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/R/utils.R b/R/utils.R index d074fda3b..088f534a1 100644 --- a/R/utils.R +++ b/R/utils.R @@ -56,25 +56,6 @@ filter_function_args <- function(fun, args) { } -#' Strip attributes from an object -#' @description This function removes all attributes from an object that are -#' specified in the `attributes` argument. -#' @param object An object to remove attributes from -#' @param attributes A character vector of attribute names to remove from the -#' object -#' @return The object with attributes removed -#' @keywords internal -strip_attributes <- function(object, attributes) { - if (is.null(object)) { - return(NULL) - } - for (i in seq_along(attributes)) { - setattr(object, attributes[i], NULL) - } - return(object) -} - - #' @title Run a function safely #' @description This is a wrapper function designed to run a function safely #' when it is not completely clear what arguments could be passed to the From e5196744fadafa5e605c8bd37d6ee3b495c9c9b9 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 23:00:23 -0600 Subject: [PATCH 13/23] update docs --- man/strip_attributes.Rd | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 man/strip_attributes.Rd diff --git a/man/strip_attributes.Rd b/man/strip_attributes.Rd deleted file mode 100644 index 5f727f539..000000000 --- a/man/strip_attributes.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils.R -\name{strip_attributes} -\alias{strip_attributes} -\title{Strip attributes from an object} -\usage{ -strip_attributes(object, attributes) -} -\arguments{ -\item{object}{An object to remove attributes from} - -\item{attributes}{A character vector of attribute names to remove from the -object} -} -\value{ -The object with attributes removed -} -\description{ -This function removes all attributes from an object that are -specified in the \code{attributes} argument. -} -\keyword{internal} From d9973c0d96d6d2509fc8ebb32e32b33872610406 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:20:36 -0600 Subject: [PATCH 14/23] Delete `plot_predictions()` from Readme to trigger CI checks first what a smart cookie I am --- README.Rmd | 21 --------------------- README.md | 25 ------------------------- 2 files changed, 46 deletions(-) diff --git a/README.Rmd b/README.Rmd index 4654fb0f7..9221faa49 100644 --- a/README.Rmd +++ b/README.Rmd @@ -97,27 +97,6 @@ remotes::install_github("epiforecasts/scoringutils", dependencies = TRUE) In this quick start guide we explore some of the functionality of the `scoringutils` package using quantile forecasts from the [ECDC forecasting hub](https://covid19forecasthub.eu/) as an example. For more detailed documentation please see the package vignettes, and individual function documentation. -### Plotting forecasts - -As a first step to evaluating the forecasts we visualise them. For the purposes of this example here we make use of `plot_predictions()` to filter the available forecasts for a single model, and forecast date. - -```{r, fig.width = 9, fig.height = 6} -example_quantile %>% - make_NA(what = "truth", - target_end_date >= "2021-07-15", - target_end_date < "2021-05-22" - ) %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != "2021-06-28" - ) %>% - plot_predictions( - x = "target_end_date", - by = c("target_type", "location") - ) + - facet_wrap(target_type ~ location, ncol = 4, scales = "free") -``` - ### Scoring forecasts Forecasts can be easily and quickly scored using the `score()` function. `score()` automatically tries to determine the `forecast_unit`, i.e. the set of columns that uniquely defines a single forecast, by taking all column names of the data into account. However, it is recommended to set the forecast unit manually by specifying the "forecast_unit" argument in `as_forecast()` as this may help to avoid errors. This will drop all columns that are neither part of the forecast unit nor part of the columns internally used by `scoringutils`. The function `as_forecast()` processes and validates the inputs. diff --git a/README.md b/README.md index 289dd4366..ba7e3c6e9 100644 --- a/README.md +++ b/README.md @@ -103,31 +103,6 @@ forecasting hub](https://covid19forecasthub.eu/) as an example. For more detailed documentation please see the package vignettes, and individual function documentation. -### Plotting forecasts - -As a first step to evaluating the forecasts we visualise them. For the -purposes of this example here we make use of `plot_predictions()` to -filter the available forecasts for a single model, and forecast date. - -``` r -example_quantile %>% - make_NA(what = "truth", - target_end_date >= "2021-07-15", - target_end_date < "2021-05-22" - ) %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != "2021-06-28" - ) %>% - plot_predictions( - x = "target_end_date", - by = c("target_type", "location") - ) + - facet_wrap(target_type ~ location, ncol = 4, scales = "free") -``` - -![](man/figures/unnamed-chunk-4-1.png) - ### Scoring forecasts Forecasts can be easily and quickly scored using the `score()` function. From 01b572be3a3f889fc7e330a9cc4028dadbadda1c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:22:50 -0600 Subject: [PATCH 15/23] delete `plot_predictions()` and `make_NA` --- NAMESPACE | 7 -- R/plot.R | 236 ---------------------------------------- man/make_NA.Rd | 40 ------- man/plot_predictions.Rd | 70 ------------ 4 files changed, 353 deletions(-) delete mode 100644 man/make_NA.Rd delete mode 100644 man/plot_predictions.Rd diff --git a/NAMESPACE b/NAMESPACE index 8df5afe85..3acaa4ee5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -47,8 +47,6 @@ export(log_shift) export(logs_binary) export(logs_sample) export(mad_sample) -export(make_NA) -export(make_na) export(merge_pred_and_obs) export(new_forecast) export(overprediction) @@ -61,7 +59,6 @@ export(plot_heatmap) export(plot_interval_coverage) export(plot_pairwise_comparison) export(plot_pit) -export(plot_predictions) export(plot_quantile_coverage) export(plot_score_table) export(plot_wis) @@ -129,7 +126,6 @@ importFrom(data.table,setDT) importFrom(data.table,setattr) importFrom(data.table,setcolorder) importFrom(data.table,setnames) -importFrom(ggdist,geom_lineribbon) importFrom(ggplot2,.data) importFrom(ggplot2,`%+replace%`) importFrom(ggplot2,aes) @@ -143,9 +139,7 @@ importFrom(ggplot2,facet_grid) importFrom(ggplot2,facet_wrap) importFrom(ggplot2,geom_col) importFrom(ggplot2,geom_histogram) -importFrom(ggplot2,geom_line) importFrom(ggplot2,geom_linerange) -importFrom(ggplot2,geom_point) importFrom(ggplot2,geom_polygon) importFrom(ggplot2,geom_text) importFrom(ggplot2,geom_tile) @@ -168,7 +162,6 @@ importFrom(ggplot2,xlab) importFrom(ggplot2,ylab) importFrom(lifecycle,deprecated) importFrom(methods,hasArg) -importFrom(rlang,enexprs) importFrom(rlang,warn) importFrom(scoringRules,crps_sample) importFrom(scoringRules,dss_sample) diff --git a/R/plot.R b/R/plot.R index a59db71f7..5b336e73d 100644 --- a/R/plot.R +++ b/R/plot.R @@ -266,242 +266,6 @@ plot_heatmap <- function(scores, return(plot) } -#' @title Plot Predictions vs True Values -#' -#' @description -#' Make a plot of observed and predicted values -#' -#' @param data a data.frame that follows the same specifications outlined in -#' [score()]. To customise your plotting, you can filter your data using the -#' function [make_NA()]. -#' @param by character vector with column names that denote categories by which -#' the plot should be stratified. If for example you want to have a facetted -#' plot, this should be a character vector with the columns used in facetting -#' (note that the facetting still needs to be done outside of the function call) -#' @param x character vector of length one that denotes the name of the variable -#' @param interval_range numeric vector indicating the interval ranges to plot. -#' If 0 is included in `interval_range`, the median prediction will be shown. -#' @return ggplot object with a plot of true vs predicted values -#' @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light -#' @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point -#' @importFrom data.table dcast -#' @importFrom ggdist geom_lineribbon -#' @export -#' @examples -#' library(ggplot2) -#' library(magrittr) -#' -#' example_continuous %>% -#' make_NA ( -#' what = "truth", -#' target_end_date >= "2021-07-22", -#' target_end_date < "2021-05-01" -#' ) %>% -#' make_NA ( -#' what = "forecast", -#' model != "EuroCOVIDhub-ensemble", -#' forecast_date != "2021-06-07" -#' ) %>% -#' plot_predictions ( -#' x = "target_end_date", -#' by = c("target_type", "location"), -#' interval_range = c(0, 50, 90, 95) -#' ) + -#' facet_wrap(~ location + target_type, scales = "free_y") + -#' aes(fill = model, color = model) -#' -#' example_continuous %>% -#' make_NA ( -#' what = "truth", -#' target_end_date >= "2021-07-22", -#' target_end_date < "2021-05-01" -#' ) %>% -#' make_NA ( -#' what = "forecast", -#' forecast_date != "2021-06-07" -#' ) %>% -#' plot_predictions ( -#' x = "target_end_date", -#' by = c("target_type", "location"), -#' interval_range = 0 -#' ) + -#' facet_wrap(~ location + target_type, scales = "free_y") + -#' aes(fill = model, color = model) - -plot_predictions <- function(data, - by = NULL, - x = "date", - interval_range = c(0, 50, 90)) { - - # split truth data and forecasts in order to apply different filtering - truth_data <- data.table::as.data.table(data)[!is.na(observed)] - forecasts <- data.table::as.data.table(data)[!is.na(predicted)] - - del_cols <- - colnames(truth_data)[!(colnames(truth_data) %in% c(by, "observed", x))] - - truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) - - # find out what type of predictions we have. convert sample based to - # interval range data - - if (test_forecast_type_is_quantile(data)) { - forecasts <- quantile_to_interval( - forecasts, - keep_quantile_col = FALSE - ) - } else if (test_forecast_type_is_sample(data)) { - forecasts <- sample_to_interval_long( - forecasts, - interval_range = interval_range, - keep_quantile_col = FALSE - ) - } - - # select appropriate boundaries and pivot wider - select <- forecasts$interval_range %in% setdiff(interval_range, 0) - intervals <- forecasts[select, ] - - # delete quantile column in intervals if present. This is important for - # pivoting - if ("quantile_level" %in% names(intervals)) { - intervals[, quantile_level := NULL] - } - - plot <- ggplot(data = data, aes(x = .data[[x]])) + - theme_scoringutils() + - ylab("True and predicted values") - - if (nrow(intervals) != 0) { - # pivot wider and convert range to a factor - intervals <- data.table::dcast(intervals, ... ~ boundary, - value.var = "predicted") - - # only plot interval ranges if there are interval ranges to plot - plot <- plot + - ggdist::geom_lineribbon( - data = intervals, - aes( - ymin = lower, ymax = upper, - # We use the fill_ramp aesthetic for this instead of the default fill - # because we want to keep fill to be able to use it for other - # variables - fill_ramp = factor( - interval_range, - levels = sort(unique(interval_range), decreasing = TRUE) - ) - ), - lwd = 0.4 - ) + - ggdist::scale_fill_ramp_discrete( - name = "interval_range", - # range argument was added to make sure that the line for the median - # and the ribbon don't have the same opacity, making the line - # invisible - range = c(0.15, 0.75) - ) - } - - # We could treat this step as part of ggdist::geom_lineribbon() but we treat - # it separately here to deal with the case when only the median is provided - # (in which case ggdist::geom_lineribbon() will fail) - if (0 %in% interval_range) { - select_median <- - forecasts$interval_range == 0 & forecasts$boundary == "lower" - median <- forecasts[select_median] - - if (nrow(median) > 0) { - plot <- plot + - geom_line( - data = median, - mapping = aes(y = predicted), - lwd = 0.4 - ) - } - } - - # add observed values - if (nrow(truth_data) > 0) { - plot <- plot + - geom_point( - data = truth_data, - show.legend = FALSE, - inherit.aes = FALSE, - aes(x = .data[[x]], y = observed), - color = "black", - size = 0.5 - ) + - geom_line( - data = truth_data, - inherit.aes = FALSE, - show.legend = FALSE, - aes(x = .data[[x]], y = observed), - linetype = 1, - color = "grey40", - lwd = 0.2 - ) - } - - return(plot) -} - -#' @title Make Rows NA in Data for Plotting -#' -#' @description -#' Filters the data and turns values into `NA` before the data gets passed to -#' [plot_predictions()]. The reason to do this is to this is that it allows to -#' 'filter' prediction and truth data separately. Any value that is NA will then -#' be removed in the subsequent call to [plot_predictions()]. -#' -#' @inheritParams score -#' @param what character vector that determines which values should be turned -#' into `NA`. If `what = "truth"`, values in the column 'observed' will be -#' turned into `NA`. If `what = "forecast"`, values in the column 'prediction' -#' will be turned into `NA`. If `what = "both"`, values in both column will be -#' turned into `NA`. -#' @param ... logical statements used to filter the data -#' @return A data.table -#' @importFrom rlang enexprs -#' @keywords plotting -#' @export -#' -#' @examples -#' make_NA ( -#' example_continuous, -#' what = "truth", -#' target_end_date >= "2021-07-22", -#' target_end_date < "2021-05-01" -#' ) - -make_NA <- function(data = NULL, - what = c("truth", "forecast", "both"), - ...) { - - assert_not_null(data = data) - - data <- data.table::copy(data) - what <- match.arg(what) - - # turn ... arguments into expressions - args <- enexprs(...) - - vars <- NULL - if (what %in% c("forecast", "both")) { - vars <- c(vars, "predicted") - } - if (what %in% c("truth", "both")) { - vars <- c(vars, "observed") - } - for (expr in args) { - data <- data[eval(expr), eval(vars) := NA_real_] - } - return(data[]) -} - -#' @rdname make_NA -#' @keywords plotting -#' @export -make_na <- make_NA #' @title Plot Interval Coverage #' diff --git a/man/make_NA.Rd b/man/make_NA.Rd deleted file mode 100644 index 1b1814c0e..000000000 --- a/man/make_NA.Rd +++ /dev/null @@ -1,40 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{make_NA} -\alias{make_NA} -\alias{make_na} -\title{Make Rows NA in Data for Plotting} -\usage{ -make_NA(data = NULL, what = c("truth", "forecast", "both"), ...) - -make_na(data = NULL, what = c("truth", "forecast", "both"), ...) -} -\arguments{ -\item{data}{A data.frame or data.table with predicted and observed values.} - -\item{what}{character vector that determines which values should be turned -into \code{NA}. If \code{what = "truth"}, values in the column 'observed' will be -turned into \code{NA}. If \code{what = "forecast"}, values in the column 'prediction' -will be turned into \code{NA}. If \code{what = "both"}, values in both column will be -turned into \code{NA}.} - -\item{...}{logical statements used to filter the data} -} -\value{ -A data.table -} -\description{ -Filters the data and turns values into \code{NA} before the data gets passed to -\code{\link[=plot_predictions]{plot_predictions()}}. The reason to do this is to this is that it allows to -'filter' prediction and truth data separately. Any value that is NA will then -be removed in the subsequent call to \code{\link[=plot_predictions]{plot_predictions()}}. -} -\examples{ -make_NA ( - example_continuous, - what = "truth", - target_end_date >= "2021-07-22", - target_end_date < "2021-05-01" - ) -} -\keyword{plotting} diff --git a/man/plot_predictions.Rd b/man/plot_predictions.Rd deleted file mode 100644 index 7fa924a9f..000000000 --- a/man/plot_predictions.Rd +++ /dev/null @@ -1,70 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/plot.R -\name{plot_predictions} -\alias{plot_predictions} -\title{Plot Predictions vs True Values} -\usage{ -plot_predictions(data, by = NULL, x = "date", interval_range = c(0, 50, 90)) -} -\arguments{ -\item{data}{a data.frame that follows the same specifications outlined in -\code{\link[=score]{score()}}. To customise your plotting, you can filter your data using the -function \code{\link[=make_NA]{make_NA()}}.} - -\item{by}{character vector with column names that denote categories by which -the plot should be stratified. If for example you want to have a facetted -plot, this should be a character vector with the columns used in facetting -(note that the facetting still needs to be done outside of the function call)} - -\item{x}{character vector of length one that denotes the name of the variable} - -\item{interval_range}{numeric vector indicating the interval ranges to plot. -If 0 is included in \code{interval_range}, the median prediction will be shown.} -} -\value{ -ggplot object with a plot of true vs predicted values -} -\description{ -Make a plot of observed and predicted values -} -\examples{ -library(ggplot2) -library(magrittr) - -example_continuous \%>\% - make_NA ( - what = "truth", - target_end_date >= "2021-07-22", - target_end_date < "2021-05-01" - ) \%>\% - make_NA ( - what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != "2021-06-07" - ) \%>\% - plot_predictions ( - x = "target_end_date", - by = c("target_type", "location"), - interval_range = c(0, 50, 90, 95) - ) + - facet_wrap(~ location + target_type, scales = "free_y") + - aes(fill = model, color = model) - -example_continuous \%>\% - make_NA ( - what = "truth", - target_end_date >= "2021-07-22", - target_end_date < "2021-05-01" - ) \%>\% - make_NA ( - what = "forecast", - forecast_date != "2021-06-07" - ) \%>\% - plot_predictions ( - x = "target_end_date", - by = c("target_type", "location"), - interval_range = 0 - ) + - facet_wrap(~ location + target_type, scales = "free_y") + - aes(fill = model, color = model) -} From 968c63b6ea2a75bcef8eb54d3d248466e9492e00 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 29 Feb 2024 04:25:30 +0000 Subject: [PATCH 16/23] Automatic readme update [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba7e3c6e9..4b1e5afe9 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ example_quantile %>% #> underprediction dispersion bias interval_coverage_50 #> #> 1: 4237.177310 3663.52458 -0.05640625 0.3906250 -#> 2: 10284.972826 4102.50094 0.09726563 0.3281250 +#> 2: 10284.972826 4102.50094 0.09726562 0.3281250 #> 3: 3260.355639 5664.37795 -0.07890625 0.4687500 #> 4: 4.103261 30.18099 0.07265625 0.8750000 #> 5: 2.098505 91.40625 0.33906250 0.6640625 From f580b930acc0d23f5aa52856d38309348793b358 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:27:33 -0600 Subject: [PATCH 17/23] Remove `plot_predictions()` from Vignette --- tests/testthat/test-plot_predictions.R | 115 ------------------------- vignettes/scoringutils.Rmd | 21 ----- 2 files changed, 136 deletions(-) delete mode 100644 tests/testthat/test-plot_predictions.R diff --git a/tests/testthat/test-plot_predictions.R b/tests/testthat/test-plot_predictions.R deleted file mode 100644 index f8220a9df..000000000 --- a/tests/testthat/test-plot_predictions.R +++ /dev/null @@ -1,115 +0,0 @@ -test_that("plot_predictions() works with point forecasts", { - d <- scoringutils::example_quantile - d <- d[d$quantile_level == 0.5 | is.na(d$quantile_level), ] - - p <- - d %>% - make_NA(what = "truth", - target_end_date <= '2021-05-01', - target_end_date > '2021-07-22') %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != '2021-06-07') %>% - plot_predictions( - by = c("location", "target_type"), - x = "target_end_date", - ) + - facet_wrap(location ~ target_type, scales = "free_y") - - expect_s3_class(p, "ggplot") - - skip_on_cran() - vdiffr::expect_doppelganger("point_forecasts", p) -}) - -test_that("plot_predictions() can handle an arbitrary number of quantiles", { - example2 <- scoringutils::example_quantile - - p <- example2 %>% - make_NA(what = "truth", - target_end_date <= '2021-05-01', - target_end_date > '2021-07-22') %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != '2021-06-07') %>% - plot_predictions( - by = c("location", "target_type"), - x = "target_end_date", - interval_range = c(0, 10, 20, 30, 40, 50, 60) - ) + - facet_wrap(location ~ target_type, scales = "free_y") - - expect_s3_class(p, "ggplot") - - skip_on_cran() - vdiffr::expect_doppelganger("many_quantiles", p) - - example1 <- scoringutils::example_continuous - - p2 <- example1 %>% - make_NA(what = "truth", - target_end_date <= '2021-05-01', - target_end_date > '2021-07-22') %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != '2021-06-07') %>% - plot_predictions( - by = c("location", "target_type"), - x = "target_end_date", - interval_range = c(0, 50, 90, 95) - ) + - facet_wrap(location ~ target_type, scales = "free_y") - expect_s3_class(p2, "ggplot") - - skip_on_cran() - vdiffr::expect_doppelganger("many_quantiles_from_sample", p2) -}) - -test_that("plot_predictions() works without median", { - - example3 <- subset( - scoringutils::example_quantile, - is.na(quantile_level) | quantile_level != 0.5 - ) - - p <- example3 %>% - make_NA(what = "truth", - target_end_date <= '2021-06-25', - target_end_date > '2021-07-12') %>% - make_NA(what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != '2021-07-12') %>% - plot_predictions( - x = "target_end_date", - by = c("location_name", "target_type") - ) + - facet_wrap(location_name ~ target_type, scales = "free_y") - - expect_s3_class(p, "ggplot") - - skip_on_cran() - vdiffr::expect_doppelganger("no_median", p) -}) - - -test_that("make_NA() works as expected", { - - example_quantile %>% - make_NA(what = "both", - target_end_date < "1999-01-01") - - example_quantile %>% - make_NA(what = "both", - 'target_end_date < "1999-01-01"') - - - expect_error(make_NA(example_quantile, what = "something wrong")) - - expect_error(make_NA()) - - expect_error( - example_quantile %>% - make_NA(what = "truth", - this_is_wrong < "1999-01-01") - ) -}) diff --git a/vignettes/scoringutils.Rmd b/vignettes/scoringutils.Rmd index 356aba5b4..f16fb4b9f 100644 --- a/vignettes/scoringutils.Rmd +++ b/vignettes/scoringutils.Rmd @@ -130,27 +130,6 @@ forecast_quantile %>% facet_wrap(~ target_type) ``` -You can also visualise forecasts directly using the `plot_predictions()` function: - -```{r, fig.width = 9, fig.height = 6} -forecast_quantile %>% - make_NA( - what = "truth", - target_end_date >= "2021-07-15", - target_end_date < "2021-05-22" - ) %>% - make_NA( - what = "forecast", - model != "EuroCOVIDhub-ensemble", - forecast_date != "2021-06-28" - ) %>% - plot_predictions( - x = "target_end_date", - by = c("target_type", "location") - ) + - facet_wrap(target_type ~ location, ncol = 4, scales = "free") -``` - ## Scoring and summarising forecasts Forecasts can easily be scored using the `score()` function. From 39542d11f66c67f7ad648d5c17290daee57a32b6 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:34:44 -0600 Subject: [PATCH 18/23] update news file --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 0a3adc1ac..8e96a1633 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,6 +49,7 @@ The update introduces breaking changes. If you want to keep using the older vers - Added a method for `print()` that prints out additional information for `forecast` objects. - Added a subsetting `[` operator for scores, so that the score name attribute gets preserved when subsetting. - Deleted the function `plot_ranges()`. If you want to continue using the functionality, you can find the function code [here](https://github.com/epiforecasts/scoringutils/issues/462). +- Removed the function `plot_predictions()` in favour of a dedicated Vignette that shows different ways of visualising predictions. For future reference, the function code can be found [here](https://github.com/epiforecasts/scoringutils/issues/659) (Issue #659). # scoringutils 1.2.2 From 74bbd1301945fcb061760a8641a3b0ea4fe2fa90 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:38:13 -0600 Subject: [PATCH 19/23] drop ggdist from imports --- DESCRIPTION | 1 - 1 file changed, 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index cb034701c..c5be3794d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -53,7 +53,6 @@ LazyData: true Imports: checkmate, data.table, - ggdist (>= 3.2.0), ggplot2 (>= 3.4.0), lifecycle, methods, From d0eba54750936527ddec82057d0deb335df747ee Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:38:26 -0600 Subject: [PATCH 20/23] Add geom_line to imports --- NAMESPACE | 1 + R/plot.R | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 3acaa4ee5..c32f0b1ec 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -139,6 +139,7 @@ importFrom(ggplot2,facet_grid) importFrom(ggplot2,facet_wrap) importFrom(ggplot2,geom_col) importFrom(ggplot2,geom_histogram) +importFrom(ggplot2,geom_line) importFrom(ggplot2,geom_linerange) importFrom(ggplot2,geom_polygon) importFrom(ggplot2,geom_text) diff --git a/R/plot.R b/R/plot.R index 5b336e73d..9dd0cbab1 100644 --- a/R/plot.R +++ b/R/plot.R @@ -278,7 +278,7 @@ plot_heatmap <- function(scores, #' Default is "model". #' @return ggplot object with a plot of interval coverage #' @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual .data -#' facet_wrap facet_grid geom_polygon +#' facet_wrap facet_grid geom_polygon geom_line #' @importFrom data.table dcast #' @export #' @examples @@ -339,7 +339,7 @@ plot_interval_coverage <- function(coverage, #' Default is "model". #' @return ggplot object with a plot of interval coverage #' @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual .data aes -#' scale_y_continuous +#' scale_y_continuous geom_line #' @importFrom data.table dcast #' @export #' @examples From 6d8eefe394a546c9c2f1eea4324111477cb0cc4a Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:38:40 -0600 Subject: [PATCH 21/23] delete obsolete test --- tests/testthat/test-make_na.R | 37 ----------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 tests/testthat/test-make_na.R diff --git a/tests/testthat/test-make_na.R b/tests/testthat/test-make_na.R deleted file mode 100644 index ff3decd97..000000000 --- a/tests/testthat/test-make_na.R +++ /dev/null @@ -1,37 +0,0 @@ -suppressMessages(library(magrittr)) - -test_that("make_na() works as expected", { - - expect_error( - example_quantile %>% - make_na(what = "both", - target_end_date < "1999-01-01"), - NA - ) - - expect_error( - example_quantile %>% - make_na(what = "both", - 'target_end_date < "1999-01-01"'), - NA - ) - - expect_error(make_na(example_quantile, what = "something wrong")) - - expect_error(make_na()) - - expect_error( - example_quantile %>% - make_NA(what = "truth", - this_is_wrong < "1999-01-01") - ) -}) - -test_that("make_NA works as make_na", { - expect_error( - example_quantile %>% - make_NA(what = "both", - target_end_date < "1999-01-01"), - NA - ) -}) \ No newline at end of file From f0ea6f8f6354b224c2ff95c2b7780b95fcdf23da Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 28 Feb 2024 22:39:41 -0600 Subject: [PATCH 22/23] Delete obsolete snapshots --- .../many-quantiles-from-sample.svg | 448 ----------------- .../plot_predictions/many-quantiles.svg | 469 ------------------ .../_snaps/plot_predictions/no-median.svg | 347 ------------- .../plot_predictions/point-forecasts.svg | 402 --------------- 4 files changed, 1666 deletions(-) delete mode 100644 tests/testthat/_snaps/plot_predictions/many-quantiles-from-sample.svg delete mode 100644 tests/testthat/_snaps/plot_predictions/many-quantiles.svg delete mode 100644 tests/testthat/_snaps/plot_predictions/no-median.svg delete mode 100644 tests/testthat/_snaps/plot_predictions/point-forecasts.svg diff --git a/tests/testthat/_snaps/plot_predictions/many-quantiles-from-sample.svg b/tests/testthat/_snaps/plot_predictions/many-quantiles-from-sample.svg deleted file mode 100644 index fafe3d4dd..000000000 --- a/tests/testthat/_snaps/plot_predictions/many-quantiles-from-sample.svg +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -IT -Cases - - - - - - - - - -IT -Deaths - - - - - - - - - - - - - - - - - - -FR -Deaths - - - - - - - - - -GB -Cases - - - - - - - - - -GB -Deaths - - - - - - - - - -DE -Cases - - - - - - - - - -DE -Deaths - - - - - - - - - -FR -Cases - - - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - --2e+05 --1e+05 -0e+00 -1e+05 - - - - - -0 -100 -200 -300 -400 - - - - - - -0 -500 -1000 -1500 - - - - - -0e+00 -1e+05 -2e+05 -3e+05 - - - - - -0 -500 -1000 -1500 - - - - - -0 -30000 -60000 -90000 - - - - - -0 -500 -1000 -1500 - - - - - -0 -20000 -40000 -60000 - - - - -target_end_date -True and predicted values -interval_range - - - - - - -95 -90 -50 -many_quantiles_from_sample - - diff --git a/tests/testthat/_snaps/plot_predictions/many-quantiles.svg b/tests/testthat/_snaps/plot_predictions/many-quantiles.svg deleted file mode 100644 index 88cbc8657..000000000 --- a/tests/testthat/_snaps/plot_predictions/many-quantiles.svg +++ /dev/null @@ -1,469 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -IT -Cases - - - - - - - - - -IT -Deaths - - - - - - - - - - - - - - - - - - -FR -Deaths - - - - - - - - - -GB -Cases - - - - - - - - - -GB -Deaths - - - - - - - - - -DE -Cases - - - - - - - - - -DE -Deaths - - - - - - - - - -FR -Cases - - - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - --2e+05 --1e+05 -0e+00 -1e+05 - - - - - -100 -200 - - - -400 -800 -1200 -1600 - - - - - -0e+00 -1e+05 -2e+05 -3e+05 - - - - - -500 -1000 -1500 - - - - -0 -30000 -60000 -90000 - - - - - -400 -800 -1200 - - - - -20000 -40000 -60000 - - - -target_end_date -True and predicted values -interval_range - - - - - - - - - - - - -60 -50 -40 -30 -20 -10 -many_quantiles - - diff --git a/tests/testthat/_snaps/plot_predictions/no-median.svg b/tests/testthat/_snaps/plot_predictions/no-median.svg deleted file mode 100644 index a3f240475..000000000 --- a/tests/testthat/_snaps/plot_predictions/no-median.svg +++ /dev/null @@ -1,347 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -United Kingdom -Cases - - - - - - - - - -United Kingdom -Deaths - - - - - - - - - - - - - - - - - - -Germany -Deaths - - - - - - - - - -Italy -Cases - - - - - - - - - -Italy -Deaths - - - - - - - - - -France -Cases - - - - - - - - - -France -Deaths - - - - - - - - - -Germany -Cases - - - - - - - -Jun 28 -Jul 05 -Jul 12 -Jul 19 - - - - - -Jun 28 -Jul 05 -Jul 12 -Jul 19 - - - - - -Jun 28 -Jul 05 -Jul 12 -Jul 19 - -5000 -10000 - - - -50 -100 -150 -200 -250 - - - - - - -100 -200 -300 -400 -500 - - - - - - -10000 -20000 -30000 - - - - -200 -400 -600 - - - - -5e+04 -1e+05 - - - -100 -200 -300 - - - - -1e+05 -2e+05 -3e+05 -4e+05 - - - - -target_end_date -True and predicted values -interval_range - - - - -90 -50 -no_median - - diff --git a/tests/testthat/_snaps/plot_predictions/point-forecasts.svg b/tests/testthat/_snaps/plot_predictions/point-forecasts.svg deleted file mode 100644 index 1d7048d2f..000000000 --- a/tests/testthat/_snaps/plot_predictions/point-forecasts.svg +++ /dev/null @@ -1,402 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -IT -Cases - - - - - - - - - -IT -Deaths - - - - - - - - - - - - - - - - - - -FR -Deaths - - - - - - - - - -GB -Cases - - - - - - - - - -GB -Deaths - - - - - - - - - -DE -Cases - - - - - - - - - -DE -Deaths - - - - - - - - - -FR -Cases - - - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - - - - - - -May 15 -Jun 01 -Jun 15 -Jul 01 -Jul 15 - --2e+05 --1e+05 -0e+00 -1e+05 - - - - - -100 -200 - - - -400 -800 -1200 -1600 - - - - - -0e+00 -1e+05 -2e+05 -3e+05 - - - - - -500 -1000 -1500 - - - - -0 -30000 -60000 -90000 - - - - - -400 -800 -1200 - - - - -20000 -40000 -60000 - - - -target_end_date -True and predicted values -point_forecasts - - From 0ca84e21786ecc06cb2eee36db4cf5698ceb6806 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 29 Feb 2024 09:01:00 -0600 Subject: [PATCH 23/23] add removal of `make_NA()` to NEWS --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8e96a1633..7d338bc72 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,7 +49,7 @@ The update introduces breaking changes. If you want to keep using the older vers - Added a method for `print()` that prints out additional information for `forecast` objects. - Added a subsetting `[` operator for scores, so that the score name attribute gets preserved when subsetting. - Deleted the function `plot_ranges()`. If you want to continue using the functionality, you can find the function code [here](https://github.com/epiforecasts/scoringutils/issues/462). -- Removed the function `plot_predictions()` in favour of a dedicated Vignette that shows different ways of visualising predictions. For future reference, the function code can be found [here](https://github.com/epiforecasts/scoringutils/issues/659) (Issue #659). +- Removed the function `plot_predictions()`, as well as its helper function `make_NA()`, in favour of a dedicated Vignette that shows different ways of visualising predictions. For future reference, the function code can be found [here](https://github.com/epiforecasts/scoringutils/issues/659) (Issue #659). # scoringutils 1.2.2