Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Elr/rel scores #69

Merged
merged 13 commits into from
Jan 7, 2025
Next Next commit
remove snapshots of hubExamples data for tests, instead directly usin…
…g hubExamples data objects
  • Loading branch information
elray1 committed Jan 3, 2025
commit 120cd25ed423a1ff4a611f9d101bcd31390b8a5d
2 changes: 1 addition & 1 deletion R/score_model_out.R
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ score_model_out <- function(model_out_tbl, oracle_output, metrics = NULL,
#' Get scoring metrics
#'
#' @param forecast A scoringutils `forecast` object (see
#' [scoringutils' general information on creating a `forecast` object][scoringutils::as_forecast_doc_template()]
#' [scoringutils' general information on creating a forecast object][scoringutils::as_forecast_doc_template()]
#' for details).
#' @inheritParams score_model_out
#'
Expand Down
4,039 changes: 0 additions & 4,039 deletions tests/testthat/helper-hubex_forecast_oracle_output.R

This file was deleted.

14,037 changes: 0 additions & 14,037 deletions tests/testthat/helper-hubex_forecast_outputs.R

This file was deleted.

66 changes: 33 additions & 33 deletions tests/testthat/test-score_model_out.R
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
test_that("score_model_out succeeds with valid inputs: mean output_type, default metrics, by all", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "mean"),
Expand Down Expand Up @@ -40,9 +40,9 @@ test_that("score_model_out succeeds with valid inputs: mean output_type, default


test_that("score_model_out succeeds with valid inputs: mean output_type, default metrics, summarize FALSE", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "mean"),
Expand Down Expand Up @@ -81,9 +81,9 @@ test_that("score_model_out succeeds with valid inputs: mean output_type, default


test_that("score_model_out succeeds with valid inputs: mean output_type, character metrics, custom by", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "mean"),
Expand Down Expand Up @@ -125,9 +125,9 @@ test_that("score_model_out succeeds with valid inputs: mean output_type, charact


test_that("score_model_out succeeds with valid inputs: median output_type, default metrics, summarize FALSE", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "median"),
Expand Down Expand Up @@ -166,9 +166,9 @@ test_that("score_model_out succeeds with valid inputs: median output_type, defau


test_that("score_model_out succeeds with valid inputs: quantile output_type, wis and interval metrics, custom by", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "quantile"),
Expand Down Expand Up @@ -248,9 +248,9 @@ test_that("score_model_out succeeds with valid inputs: quantile output_type, wis


test_that("score_model_out succeeds with valid inputs: quantile output_type, wis/interval metrics, summarize FALSE", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "quantile"),
Expand Down Expand Up @@ -322,9 +322,9 @@ test_that("score_model_out succeeds with valid inputs: quantile output_type, wis


test_that("score_model_out succeeds with valid inputs: nominal pmf output_type, default metrics, custom by", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_scores <- score_model_out(
model_out_tbl = forecast_outputs |> dplyr::filter(.data[["output_type"]] == "pmf"),
Expand Down Expand Up @@ -365,9 +365,9 @@ test_that("score_model_out succeeds with valid inputs: nominal pmf output_type,


test_that("score_model_out errors when model_out_tbl has multiple output_types", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

expect_error(
score_model_out(
Expand All @@ -380,9 +380,9 @@ test_that("score_model_out errors when model_out_tbl has multiple output_types",


test_that("score_model_out works with all kinds of interval levels are requested", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

expect_error(
score_model_out(
Expand Down Expand Up @@ -425,9 +425,9 @@ test_that("score_model_out works with all kinds of interval levels are requested


test_that("score_model_out errors when invalid metrics are requested", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

expect_error(
score_model_out(
Expand Down Expand Up @@ -473,9 +473,9 @@ test_that("score_model_out errors when invalid metrics are requested", {


test_that("score_model_out errors when an unsupported output_type is provided", {
# Forecast data from HubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubex_forecast_outputs()
forecast_oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

expect_error(
score_model_out(
Expand Down
15 changes: 7 additions & 8 deletions tests/testthat/test-transform_point_model_out.R
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,16 @@ test_that("many-to-one relationship exists between model_out_tbl and oracle_outp
})

test_that("hubExamples data set is transformed correctly", {
# forecast_outputs.rda & forecast_oracle_output.rda are stored in hubExamples:
# https://github.com/hubverse-org/hubExamples/tree/main
model_out_tbl <- hubex_forecast_outputs()
oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output
act_forecast <- transform_point_model_out(
model_out_tbl = model_out_tbl,
oracle_output = oracle_output,
model_out_tbl = forecast_outputs,
oracle_output = forecast_oracle_output,
output_type = "mean"
)

exp_forecast <- model_out_tbl |>
exp_forecast <- forecast_outputs |>
dplyr::filter(output_type == "mean") |>
dplyr::rename(model = model_id)

Expand All @@ -153,7 +152,7 @@ test_that("hubExamples data set is transformed correctly", {
# correct observed values, in alignment with oracle_output
exp_act_forecast <- dplyr::left_join(
act_forecast,
oracle_output |> dplyr::filter(output_type == "mean"),
forecast_oracle_output |> dplyr::filter(output_type == "mean"),
by = c("target", "location", "target_end_date")
)
expect_equal(exp_act_forecast$observed, exp_act_forecast$oracle_value)
Expand Down
16 changes: 8 additions & 8 deletions tests/testthat/test-transform_quantile_model_out.R
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,16 @@ test_that("many-to-one relationship exists between model_out_tbl and oracle_outp
})

test_that("hubExamples data set is transformed correctly", {
# forecast_outputs.rda & forecast_oracle_output.rda are stored in hubExamples:
# https://github.com/hubverse-org/hubExamples/tree/main
model_out_tbl <- hubex_forecast_outputs()
oracle_output <- hubex_forecast_oracle_output()
# Forecast data from hubExamples: <https://hubverse-org.github.io/hubExamples/reference/forecast_data.html>
forecast_outputs <- hubExamples::forecast_outputs
forecast_oracle_output <- hubExamples::forecast_oracle_output

act_forecast <- transform_quantile_model_out(
model_out_tbl = model_out_tbl,
oracle_output = oracle_output
model_out_tbl = forecast_outputs,
oracle_output = forecast_oracle_output
)

exp_forecast <- model_out_tbl |>
exp_forecast <- forecast_outputs |>
dplyr::filter(output_type == "quantile") |>
dplyr::rename(model = model_id, quantile_level = output_type_id) |>
dplyr::mutate(quantile_level = as.numeric(quantile_level))
Expand All @@ -103,7 +103,7 @@ test_that("hubExamples data set is transformed correctly", {
# correct observed values, in alignment with oracle_output
exp_act_forecast <- dplyr::left_join(
act_forecast,
oracle_output |>
forecast_oracle_output |>
dplyr::filter(output_type == "quantile") |>
dplyr::rename(quantile_level = output_type_id) |>
dplyr::mutate(quantile_level = as.numeric(quantile_level)),
Expand Down
29 changes: 0 additions & 29 deletions tests/testthat/testdata/make_hubExamples_snapshots.R

This file was deleted.

Loading