From 7512839f9d93ef50f21acf2135532fff3aad818d Mon Sep 17 00:00:00 2001 From: mrustl Date: Wed, 1 Jun 2022 13:36:22 +0200 Subject: [PATCH 01/12] bump version to development version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 838036bf..8abba574 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: wasserportal Title: R Package with Functions for Scraping Data of Wasserportal Berlin -Version: 0.1.0 +Version: 0.1.0.9000 Authors@R: c(person(given = "Hauke", family = "Sonnenberg", From fe39e58227a4bf82e6710f2ee485f93216129769 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 2 Jun 2022 16:17:04 +0200 Subject: [PATCH 02/12] Start adapt funcs for scraping SW stations (#21) --- NAMESPACE | 2 ++ R/.test-surface-water_download.R | 31 +++++++++++++++++++++++++++ R/read_wasserportal.R | 35 +++++++++++++++++++------------ R/read_wasserportal_raw.R | 36 +++++++++++++++++++++++++++----- man/read_wasserportal.Rd | 24 +++++++++++++-------- man/read_wasserportal_raw.Rd | 6 +++++- 6 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 R/.test-surface-water_download.R diff --git a/NAMESPACE b/NAMESPACE index 0d5d688f..7d2a0efc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ export("%>%") export(columns_to_labels) export(get_overview_options) +export(get_station_variables) export(get_stations) export(get_wasserportal_master_data) export(get_wasserportal_masters_data) @@ -22,6 +23,7 @@ importFrom(dplyr,left_join) importFrom(dplyr,mutate) importFrom(dplyr,rename) importFrom(dplyr,select) +importFrom(dplyr,select_if) importFrom(httr,POST) importFrom(httr,content) importFrom(kwb.datetime,textToEuropeBerlinPosix) diff --git a/R/.test-surface-water_download.R b/R/.test-surface-water_download.R new file mode 100644 index 00000000..3758f44d --- /dev/null +++ b/R/.test-surface-water_download.R @@ -0,0 +1,31 @@ +if (FALSE) { +library(wasserportal) + +stations <- wasserportal::get_stations() + +stations_crosstable <- stations$crosstable + +station_crosstable <- stations_crosstable[1,] + + +read_wasserportal_raw( + station = station_crosstable$Messstellennummer, + variable = get_station_variables(station_crosstable)[2], + type = "daily", + from_date = from_date, + include_raw_time = TRUE, + stations_crosstable = stations_crosstable +) + + +sw_daily <- stats::setNames(lapply(stations_crosstable$Messstellennummer, + function(station) { + read_wasserportal( + station = station, + type = "daily", + from_date = from_date, + include_raw_time = TRUE, + stations_crosstable = stations_crosstable +)} +), nm = stations_crosstable$Messstellennummer) +} diff --git a/R/read_wasserportal.R b/R/read_wasserportal.R index f2f7d32d..b05266be 100644 --- a/R/read_wasserportal.R +++ b/R/read_wasserportal.R @@ -13,9 +13,9 @@ #' \code{UTCOffset}, 1 in winter, 2 in summer). #' #' @param station station number, as returned by -#' \code{\link{get_wasserportal_stations}} +#' \code{\link{get_stations}} #' @param variables vector of variable identifiers, as returned by -#' \code{\link{get_wasserportal_variables}} +#' \code{\link{get_station_variables}} #' @param from_date \code{Date} object (or string in format "yyyy-mm-dd" that #' can be converted to a \code{Date} object representing the first day for #' which to request data. Default: \code{as.character(Sys.Date() - 90L)} @@ -23,6 +23,8 @@ #' @param include_raw_time if \code{TRUE} the original time column and the #' column with the corrected winter time are included in the output. The #' default is \code{FALSE}. +#' @param stations_crosstable sublist `crosstable` as retrieved from \link{\code{get_stations}} +#' i.e. `get_stations()$crosstable` #' @return data frame read from the CSV file that the download provides. #' IMPORTANT: It is not yet clear how to interpret the timestamp, see example #' @importFrom httr POST content @@ -30,17 +32,19 @@ #' @export #' @examples #' # Get a list of available water quality stations and variables -#' stations <- wasserportal::get_wasserportal_stations() -#' variables <- wasserportal::get_wasserportal_variables() +#' stations <- wasserportal::get_stations() +#' stations_crosstable <- stations$crosstable +#' station_crosstable <- stations_crosstable[1,] #' #' # Set the start date #' from_date <- "2021-03-01" #' #' # Read the timeseries (multiple variables for one station) #' water_quality <- wasserportal::read_wasserportal( -#' station = stations$MPS_Charlottenburg, -#' variables = c(variables["Sauerstoffgehalt"], variables["Leitfaehigkeit"]), -#' from_date = from_date, include_raw_time = TRUE +#' station = station_crosstable$Messstellennummer, +#' from_date = from_date, +#' include_raw_time = TRUE, +#' stations_crosstable = stations_crosstable #' ) #' #' # Look at the first few records @@ -75,17 +79,20 @@ #' sum(water_quality$timestamp_raw != water_quality$timestamp_corr) #' read_wasserportal <- function( - station, variables = get_wasserportal_variables(station), + station, + variables = NULL, from_date = as.character(Sys.Date() - 90L), type = "single", - include_raw_time = FALSE + include_raw_time = FALSE, + stations_crosstable ) { #kwb.utils::assignPackageObjects("wasserportal") #station=get_wasserportal_stations(type = "flow")$Tiefwerder #variables = get_wasserportal_variables(station);from_date = "2019-01-01";include_raw_time = FALSE - - variable_ids <- get_wasserportal_variables() - station_ids <- get_wasserportal_stations(type = NULL) + station_crosstable <- stations_crosstable[stations_crosstable$Messstellennummer == station,] + variable_ids <- get_station_variables(station_crosstable) + if(is.null(variables)) variables <- variable_ids + station_ids <- stations_crosstable[["Messstellennummer"]] stopifnot(all(station %in% station_ids)) stopifnot(all(variables %in% variable_ids)) @@ -101,7 +108,9 @@ read_wasserportal <- function( from_date = from_date, type = type, include_raw_time = include_raw_time, - handle = handle + handle = handle, + stations_crosstable = stations_crosstable + ) # Remove elements of class "response" that are returned in case of an error diff --git a/R/read_wasserportal_raw.R b/R/read_wasserportal_raw.R index 00490479..cd3ffa73 100644 --- a/R/read_wasserportal_raw.R +++ b/R/read_wasserportal_raw.R @@ -1,3 +1,18 @@ +#' Helper function: get available station variables +#' +#' @param station_df station_df +#' +#' @return returns names of available variables for station +#' @export +#' +#' @importFrom dplyr select_if +#' +get_station_variables <- function(station_df) { + station_df <- station_df %>% + dplyr::select_if(function(x){!all(is.na(x))}) + names(station_df)[!names(station_df) %in% c("Messstellennummer", "Messstellenname")] +} + # read_wasserportal_raw -------------------------------------------------------- #' Read Wasserportal Raw @@ -8,31 +23,42 @@ #' @param type one of "single", "daily", "monthly" (default: "single") #' @param include_raw_time TRUE or FALSE (default: FALSE) #' @param handle handle (default: NULL) -#' +#' @param stations_crosstable sublist `crosstable` as retrieved from \link{\code{get_stations}} +#' i.e. `get_stations()$crosstable` #' @return ???? #' @export #' @import kwb.utils #' @importFrom kwb.datetime textToEuropeBerlinPosix read_wasserportal_raw <- function( variable, station, from_date, type = "single", include_raw_time = FALSE, - handle = NULL + handle = NULL, + stations_crosstable ) { #variable <- variables[1] from_date <- assert_date(from_date) + stopifnot(length(station) == 1) - station_ids <- get_wasserportal_stations(type = NULL) + station_ids <- stations_crosstable$Messstellennummer stopifnot(station %in% station_ids) stopifnot(length(variable) == 1) - variable_ids <- get_wasserportal_variables(station) + + station_df <- stations_crosstable[stations_crosstable$Messstellennummer == station, ] %>% + dplyr::select_if(function(x){!all(is.na(x))}) + + variable_ids <- get_station_variables(station_df) stopifnot(variable %in% variable_ids) sreihe <- kwb.utils::selectElements(elements = type, list( single = "w", single_all = "wa", daily = "m", monthly = "j" )) + variable <- kwb.utils::selectElements(elements = variable, list( + ws = "w", df = "d", wt = "t", lf = "l", ph = "p", og = "g", os = "s" + )) + progress <- get_wasserportal_text(station, variable, station_ids, variable_ids) url <- get_wasserportal_url(station, variable) @@ -71,7 +97,7 @@ read_wasserportal_raw <- function( data <- read(text, header = FALSE, skip = 1) # Get the numbers of the data columns - if (type != "monthly") { + if (!type %in% c("daily", "monthly")) { stopifnot(ncol(data) == 2L) } diff --git a/man/read_wasserportal.Rd b/man/read_wasserportal.Rd index 1484dc5e..6eda365b 100644 --- a/man/read_wasserportal.Rd +++ b/man/read_wasserportal.Rd @@ -6,18 +6,19 @@ \usage{ read_wasserportal( station, - variables = get_wasserportal_variables(station), + variables = NULL, from_date = as.character(Sys.Date() - 90L), type = "single", - include_raw_time = FALSE + include_raw_time = FALSE, + stations_crosstable ) } \arguments{ \item{station}{station number, as returned by -\code{\link{get_wasserportal_stations}}} +\code{\link{get_stations}}} \item{variables}{vector of variable identifiers, as returned by -\code{\link{get_wasserportal_variables}}} +\code{\link{get_station_variables}}} \item{from_date}{\code{Date} object (or string in format "yyyy-mm-dd" that can be converted to a \code{Date} object representing the first day for @@ -28,6 +29,9 @@ which to request data. Default: \code{as.character(Sys.Date() - 90L)}} \item{include_raw_time}{if \code{TRUE} the original time column and the column with the corrected winter time are included in the output. The default is \code{FALSE}.} + +\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \link{\code{get_stations}} +i.e. \code{get_stations()$crosstable}} } \value{ data frame read from the CSV file that the download provides. @@ -47,17 +51,19 @@ together with the additional information on the UTC offset (column } \examples{ # Get a list of available water quality stations and variables -stations <- wasserportal::get_wasserportal_stations() -variables <- wasserportal::get_wasserportal_variables() +stations <- wasserportal::get_stations() +stations_crosstable <- stations$crosstable +station_crosstable <- stations_crosstable[1,] # Set the start date from_date <- "2021-03-01" # Read the timeseries (multiple variables for one station) water_quality <- wasserportal::read_wasserportal( - station = stations$MPS_Charlottenburg, - variables = c(variables["Sauerstoffgehalt"], variables["Leitfaehigkeit"]), - from_date = from_date, include_raw_time = TRUE + station = station_crosstable$Messstellennummer, + from_date = from_date, + include_raw_time = TRUE, + stations_crosstable = stations_crosstable ) # Look at the first few records diff --git a/man/read_wasserportal_raw.Rd b/man/read_wasserportal_raw.Rd index 8c923392..ccc6fc49 100644 --- a/man/read_wasserportal_raw.Rd +++ b/man/read_wasserportal_raw.Rd @@ -10,7 +10,8 @@ read_wasserportal_raw( from_date, type = "single", include_raw_time = FALSE, - handle = NULL + handle = NULL, + stations_crosstable ) } \arguments{ @@ -25,6 +26,9 @@ read_wasserportal_raw( \item{include_raw_time}{TRUE or FALSE (default: FALSE)} \item{handle}{handle (default: NULL)} + +\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \link{\code{get_stations}} +i.e. \code{get_stations()$crosstable}} } \value{ ???? From 0ec82eb83a9927fdc32a52d5c78561bb2437eeba Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 2 Jun 2022 18:24:07 +0200 Subject: [PATCH 03/12] fix bad link text https://github.com/KWB-R/wasserportal/runs/6710055489?check_suite_focus=true#step:9:190 --- R/read_wasserportal.R | 2 +- R/read_wasserportal_raw.R | 2 +- man/read_wasserportal.Rd | 2 +- man/read_wasserportal_raw.Rd | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/read_wasserportal.R b/R/read_wasserportal.R index b05266be..e92c0181 100644 --- a/R/read_wasserportal.R +++ b/R/read_wasserportal.R @@ -23,7 +23,7 @@ #' @param include_raw_time if \code{TRUE} the original time column and the #' column with the corrected winter time are included in the output. The #' default is \code{FALSE}. -#' @param stations_crosstable sublist `crosstable` as retrieved from \link{\code{get_stations}} +#' @param stations_crosstable sublist `crosstable` as retrieved from \code{\link{get_stations}} #' i.e. `get_stations()$crosstable` #' @return data frame read from the CSV file that the download provides. #' IMPORTANT: It is not yet clear how to interpret the timestamp, see example diff --git a/R/read_wasserportal_raw.R b/R/read_wasserportal_raw.R index cd3ffa73..088c4635 100644 --- a/R/read_wasserportal_raw.R +++ b/R/read_wasserportal_raw.R @@ -23,7 +23,7 @@ get_station_variables <- function(station_df) { #' @param type one of "single", "daily", "monthly" (default: "single") #' @param include_raw_time TRUE or FALSE (default: FALSE) #' @param handle handle (default: NULL) -#' @param stations_crosstable sublist `crosstable` as retrieved from \link{\code{get_stations}} +#' @param stations_crosstable sublist `crosstable` as retrieved from \code{\link{get_stations}} #' i.e. `get_stations()$crosstable` #' @return ???? #' @export diff --git a/man/read_wasserportal.Rd b/man/read_wasserportal.Rd index 6eda365b..fd0e24c5 100644 --- a/man/read_wasserportal.Rd +++ b/man/read_wasserportal.Rd @@ -30,7 +30,7 @@ which to request data. Default: \code{as.character(Sys.Date() - 90L)}} column with the corrected winter time are included in the output. The default is \code{FALSE}.} -\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \link{\code{get_stations}} +\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \code{\link{get_stations}} i.e. \code{get_stations()$crosstable}} } \value{ diff --git a/man/read_wasserportal_raw.Rd b/man/read_wasserportal_raw.Rd index ccc6fc49..bff29da2 100644 --- a/man/read_wasserportal_raw.Rd +++ b/man/read_wasserportal_raw.Rd @@ -27,7 +27,7 @@ read_wasserportal_raw( \item{handle}{handle (default: NULL)} -\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \link{\code{get_stations}} +\item{stations_crosstable}{sublist \code{crosstable} as retrieved from \code{\link{get_stations}} i.e. \code{get_stations()$crosstable}} } \value{ From 85c747290f295c370c437cbec35500a1e1e50778 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 2 Jun 2022 18:25:30 +0200 Subject: [PATCH 04/12] add missing :book: --- man/get_station_variables.Rd | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 man/get_station_variables.Rd diff --git a/man/get_station_variables.Rd b/man/get_station_variables.Rd new file mode 100644 index 00000000..8cf4218b --- /dev/null +++ b/man/get_station_variables.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_wasserportal_raw.R +\name{get_station_variables} +\alias{get_station_variables} +\title{Helper function: get available station variables} +\usage{ +get_station_variables(station_df) +} +\arguments{ +\item{station_df}{station_df} +} +\value{ +returns names of available variables for station +} +\description{ +Helper function: get available station variables +} From 5d622145d123493ec8491a321225e8a7d35f0f24 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 2 Jun 2022 18:41:25 +0200 Subject: [PATCH 05/12] catch error if master data not available --- R/get_wasserportal_master_data.R | 14 +++++++++++++- man/get_wasserportal_master_data.Rd | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/R/get_wasserportal_master_data.R b/R/get_wasserportal_master_data.R index 649e9e5d..99fc6c3d 100644 --- a/R/get_wasserportal_master_data.R +++ b/R/get_wasserportal_master_data.R @@ -18,8 +18,11 @@ wasserportal_base_url <- function() { #' @importFrom rlang .data #' @export #' @examples +#' ## GW Station #' get_wasserportal_master_data(station_id = 149) #' +#' ## SW Station +#' get_wasserportal_master_data(station_id = 5865900) get_wasserportal_master_data <- function ( station_id, @@ -35,11 +38,20 @@ get_wasserportal_master_data <- function ( master_table <- html_overview %>% rvest::html_node(xpath = '//*[@summary="Pegel Berlin"]') %>% - rvest::html_table() %>% + rvest::html_table() + + if(nrow(master_table) == 0) { + msg <- sprintf("No master table for station '%s' available at '%s'", + station_id, + master_url) + stop(msg) + } else { + master_table <- master_table %>% dplyr::rename("key" = "X1", "value" = "X2") %>% dplyr::mutate(key = stringr::str_remove_all(.data$key, "-")) %>% dplyr::mutate(key = kwb.utils::substSpecialChars(.data$key)) %>% tidyr::pivot_wider(names_from = "key", values_from = "value") + } master_table diff --git a/man/get_wasserportal_master_data.Rd b/man/get_wasserportal_master_data.Rd index 4fdd95ec..44130696 100644 --- a/man/get_wasserportal_master_data.Rd +++ b/man/get_wasserportal_master_data.Rd @@ -22,6 +22,9 @@ data frame with metadata for Wasserportal Berlin: get master data for a single station } \examples{ +## GW Station get_wasserportal_master_data(station_id = 149) +## SW Station +get_wasserportal_master_data(station_id = 5865900) } From aad79c7b429dfb1bc3130ab291478ba678ee00e2 Mon Sep 17 00:00:00 2001 From: Michael Rustler Date: Thu, 2 Jun 2022 19:14:57 +0200 Subject: [PATCH 06/12] Do not request GW master data --- R/get_wasserportal_master_data.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/get_wasserportal_master_data.R b/R/get_wasserportal_master_data.R index 99fc6c3d..97e2accd 100644 --- a/R/get_wasserportal_master_data.R +++ b/R/get_wasserportal_master_data.R @@ -19,7 +19,7 @@ wasserportal_base_url <- function() { #' @export #' @examples #' ## GW Station -#' get_wasserportal_master_data(station_id = 149) +#' # get_wasserportal_master_data(station_id = 149) #' #' ## SW Station #' get_wasserportal_master_data(station_id = 5865900) From 10005f1526917a38dc022e9d20a002c0d3b34dab Mon Sep 17 00:00:00 2001 From: Michael Rustler Date: Thu, 2 Jun 2022 19:14:57 +0200 Subject: [PATCH 07/12] Do not request GW master data --- R/get_wasserportal_master_data.R | 9 +++++---- R/read_wasserportal_raw_gw.R | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/R/get_wasserportal_master_data.R b/R/get_wasserportal_master_data.R index 99fc6c3d..e2f3ceef 100644 --- a/R/get_wasserportal_master_data.R +++ b/R/get_wasserportal_master_data.R @@ -18,11 +18,12 @@ wasserportal_base_url <- function() { #' @importFrom rlang .data #' @export #' @examples -#' ## GW Station +#'## GW Station +#'\dontrun{ #' get_wasserportal_master_data(station_id = 149) -#' -#' ## SW Station -#' get_wasserportal_master_data(station_id = 5865900) +#'} +#'## SW Station +#'get_wasserportal_master_data(station_id = 5865900) get_wasserportal_master_data <- function ( station_id, diff --git a/R/read_wasserportal_raw_gw.R b/R/read_wasserportal_raw_gw.R index 8e351186..aff0ee5a 100644 --- a/R/read_wasserportal_raw_gw.R +++ b/R/read_wasserportal_raw_gw.R @@ -15,8 +15,10 @@ #' @importFrom tidyr pivot_longer #' @importFrom dplyr select filter mutate #' @examples +#' \dontrun{ #' read_wasserportal_raw_gw(station = 149, stype = "gwl") #' read_wasserportal_raw_gw(station = 149, stype = "gwq") +#' } read_wasserportal_raw_gw <- function( station = 149, stype = "gwl", From 04165494571a89b7a19c128a6deb823141985277 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 2 Jun 2022 20:16:03 +0200 Subject: [PATCH 08/12] deactivate GW workflows as long as Wasserportal data is not available for GW stations --- R/.test-surface-water_download.R | 36 +++++++--- man/get_wasserportal_master_data.Rd | 3 +- man/read_wasserportal_raw_gw.Rd | 2 + .../{groundwater.Rmd => .groundwater.Rmd} | 0 ...-lists.Rmd => .promisces_norman-lists.Rmd} | 0 ...reach-uba.Rmd => .promisces_reach-uba.Rmd} | 0 vignettes/{tutorial.Rmd => .tutorial.Rmd} | 0 vignettes/surface-water.Rmd | 70 +++++++++++++++++++ 8 files changed, 102 insertions(+), 9 deletions(-) rename vignettes/{groundwater.Rmd => .groundwater.Rmd} (100%) rename vignettes/{promisces_norman-lists.Rmd => .promisces_norman-lists.Rmd} (100%) rename vignettes/{promisces_reach-uba.Rmd => .promisces_reach-uba.Rmd} (100%) rename vignettes/{tutorial.Rmd => .tutorial.Rmd} (100%) create mode 100644 vignettes/surface-water.Rmd diff --git a/R/.test-surface-water_download.R b/R/.test-surface-water_download.R index 3758f44d..0df84800 100644 --- a/R/.test-surface-water_download.R +++ b/R/.test-surface-water_download.R @@ -2,30 +2,50 @@ if (FALSE) { library(wasserportal) stations <- wasserportal::get_stations() +stations_crosstable <- stations$crosstable -stations_crosstable <- stations$crosstable +stations_crosstable_bb <- stations_crosstable %>% + dplyr::filter(stringr::str_detect(.data$Messstellennummer, + pattern = "^[A-Z]{2}_")) -station_crosstable <- stations_crosstable[1,] +stations_crosstable_berlin <- stations_crosstable %>% + dplyr::filter(stringr::str_detect(.data$Messstellennummer, + pattern = "^[A-Z]{2}_", + negate = TRUE)) -read_wasserportal_raw( - station = station_crosstable$Messstellennummer, - variable = get_station_variables(station_crosstable)[2], + +station_crosstable_berlin <- stations_crosstable_berlin[1,] +stations_crosstable_berlin +from_date <- "1900-01-01" +sw_station_berlin_daily <- wasserportal::read_wasserportal_raw( + station = station_crosstable_berlin$Messstellennummer, + variable = get_station_variables(stations_crosstable_berlin)[1], type = "daily", from_date = from_date, include_raw_time = TRUE, stations_crosstable = stations_crosstable ) +str(sw_station_berlin_daily) + -sw_daily <- stats::setNames(lapply(stations_crosstable$Messstellennummer, + +sw_stations_berlin_daily <- stats::setNames(lapply(stations_crosstable_berlin$Messstellennummer, function(station) { - read_wasserportal( + msg <- sprintf("Fetching data for station '%s'", station) + kwb.utils::catAndRun(msg, expr = { + wasserportal::read_wasserportal( station = station, type = "daily", from_date = from_date, include_raw_time = TRUE, stations_crosstable = stations_crosstable -)} +)})} ), nm = stations_crosstable$Messstellennummer) + +str(sw_stations_daily) + + } + diff --git a/man/get_wasserportal_master_data.Rd b/man/get_wasserportal_master_data.Rd index 44130696..d9f3409d 100644 --- a/man/get_wasserportal_master_data.Rd +++ b/man/get_wasserportal_master_data.Rd @@ -23,8 +23,9 @@ Wasserportal Berlin: get master data for a single station } \examples{ ## GW Station +\dontrun{ get_wasserportal_master_data(station_id = 149) - +} ## SW Station get_wasserportal_master_data(station_id = 5865900) } diff --git a/man/read_wasserportal_raw_gw.Rd b/man/read_wasserportal_raw_gw.Rd index fabb0fb3..da12681c 100644 --- a/man/read_wasserportal_raw_gw.Rd +++ b/man/read_wasserportal_raw_gw.Rd @@ -33,6 +33,8 @@ data.frame with values (currently only if stype == "gwl") read_wasserportal_raw_gw } \examples{ +\dontrun{ read_wasserportal_raw_gw(station = 149, stype = "gwl") read_wasserportal_raw_gw(station = 149, stype = "gwq") } +} diff --git a/vignettes/groundwater.Rmd b/vignettes/.groundwater.Rmd similarity index 100% rename from vignettes/groundwater.Rmd rename to vignettes/.groundwater.Rmd diff --git a/vignettes/promisces_norman-lists.Rmd b/vignettes/.promisces_norman-lists.Rmd similarity index 100% rename from vignettes/promisces_norman-lists.Rmd rename to vignettes/.promisces_norman-lists.Rmd diff --git a/vignettes/promisces_reach-uba.Rmd b/vignettes/.promisces_reach-uba.Rmd similarity index 100% rename from vignettes/promisces_reach-uba.Rmd rename to vignettes/.promisces_reach-uba.Rmd diff --git a/vignettes/tutorial.Rmd b/vignettes/.tutorial.Rmd similarity index 100% rename from vignettes/tutorial.Rmd rename to vignettes/.tutorial.Rmd diff --git a/vignettes/surface-water.Rmd b/vignettes/surface-water.Rmd new file mode 100644 index 00000000..090dd216 --- /dev/null +++ b/vignettes/surface-water.Rmd @@ -0,0 +1,70 @@ +--- +title: "Surface Water" +author: "Michael Rustler" +date: "`r Sys.time()`" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Surface Water} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r } +library(wasserportal) + +stations <- wasserportal::get_stations() +stations_crosstable <- stations$crosstable + +stations_crosstable_bb <- stations_crosstable %>% + dplyr::filter(stringr::str_detect(.data$Messstellennummer, + pattern = "^[A-Z]{2}_")) +stations_crosstable_bb + +stations_crosstable_berlin <- stations_crosstable %>% + dplyr::filter(stringr::str_detect(.data$Messstellennummer, + pattern = "^[A-Z]{2}_", + negate = TRUE)) +stations_crosstable_berlin + + +station_crosstable_berlin <- stations_crosstable_berlin[1,] +stations_crosstable_berlin +from_date <- "1900-01-01" +sw_station_berlin_daily <- wasserportal::read_wasserportal_raw( + station = station_crosstable_berlin$Messstellennummer, + variable = get_station_variables(stations_crosstable_berlin)[1], + type = "daily", + from_date = from_date, + include_raw_time = TRUE, + stations_crosstable = stations_crosstable_berlin +) + +str(sw_station_berlin_daily) + + + +stations_berlin <- stations_crosstable_berlin$Messstellennummer[1:5] + +sw_stations_berlin_daily <- stats::setNames(lapply(stations_berlin, + function(station) { + msg <- sprintf("Fetching data for station '%s'", station) + kwb.utils::catAndRun(msg, expr = { + wasserportal::read_wasserportal( + station = station, + type = "daily", + from_date = from_date, + include_raw_time = TRUE, + stations_crosstable = stations_crosstable +)})} +), nm = stations_berlin) + +str(sw_stations_daily) + +``` From 561db5cf1d6526afa0e4b702ce8b1a4b452f226c Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 9 Jun 2022 09:08:53 +0200 Subject: [PATCH 09/12] Prepare bugfix release v0.1.1 --- DESCRIPTION | 2 +- NEWS.md | 5 +++++ R/get_wasserportal_stations_table.R | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 8abba574..6780b8cd 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: wasserportal Title: R Package with Functions for Scraping Data of Wasserportal Berlin -Version: 0.1.0.9000 +Version: 0.1.1 Authors@R: c(person(given = "Hauke", family = "Sonnenberg", diff --git a/NEWS.md b/NEWS.md index 8a9becf7..3ad116ab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +# [wasserportal 0.1.1](https://github.com/KWB-R/wasserportal/releases/tag/v0.1.1) 2022-06-09 + +* Fix bug in `get_wasserportal_stations_table()` now correctly naming parameter +`temperature` (formerly incorrectly `level`) + # [wasserportal 0.1.0](https://github.com/KWB-R/wasserportal/releases/tag/v0.1.0) 2022-06-01 R package for scraping `groundwater` data (`groundwater level` and `quality`) from [Wasserportal Berlin](https://wasserportal.berlin.de). Please note that the diff --git a/R/get_wasserportal_stations_table.R b/R/get_wasserportal_stations_table.R index 9fad5e91..e8a72b70 100644 --- a/R/get_wasserportal_stations_table.R +++ b/R/get_wasserportal_stations_table.R @@ -11,7 +11,7 @@ get_overview_options <- function () { list(surface_water = list(water_level = "ws", flow = "df", - level = "wt", + temperature = "wt", conductivity = "lf", ph = "ph", oxygen_concentration = "og", From 0c7b87ec6137b2b76f20fe5bdeb6b1cd061011fe Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 9 Jun 2022 09:29:14 +0200 Subject: [PATCH 10/12] Fix surface water :book: --- vignettes/surface-water.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/surface-water.Rmd b/vignettes/surface-water.Rmd index 090dd216..a4d3474e 100644 --- a/vignettes/surface-water.Rmd +++ b/vignettes/surface-water.Rmd @@ -65,6 +65,6 @@ sw_stations_berlin_daily <- stats::setNames(lapply(stations_berlin, )})} ), nm = stations_berlin) -str(sw_stations_daily) +str(sw_stations_berlin_daily) ``` From cb46234cc3a4c7542258576cb218c6537e096c89 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 9 Jun 2022 09:43:43 +0200 Subject: [PATCH 11/12] zenodo_badge: cite always latest release --- NEWS.md | 2 ++ README.md | 2 +- index.md | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3ad116ab..06786cf1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ * Fix bug in `get_wasserportal_stations_table()` now correctly naming parameter `temperature` (formerly incorrectly `level`) +* Fix [Surface Water](../articles/surface-water.html) article +* Adapt Zenodo DOI badge to cite always latest release # [wasserportal 0.1.0](https://github.com/KWB-R/wasserportal/releases/tag/v0.1.0) 2022-06-01 diff --git a/README.md b/README.md index 8a3a71ba..4a56f0c7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Project Status](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://www.tidyverse.org/lifecycle/#experimental) [![CRAN_Status_Badge](https://www.r-pkg.org/badges/version/wasserportal)]() [![R-Universe_Status_Badge](https://kwb-r.r-universe.dev/badges/wasserportal)](https://kwb-r.r-universe.dev/) -[![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.6602574.svg)](https://doi.org/10.5281/zenodo.6602574) +[![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.6602573.svg)](https://doi.org/10.5281/zenodo.6602573) # wasserportal diff --git a/index.md b/index.md index f8ea0882..6937ff21 100644 --- a/index.md +++ b/index.md @@ -4,7 +4,7 @@ [![Project Status](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://www.tidyverse.org/lifecycle/#experimental) [![CRAN_Status_Badge](https://www.r-pkg.org/badges/version/wasserportal)]() [![R-Universe_Status_Badge](https://kwb-r.r-universe.dev/badges/wasserportal)](https://kwb-r.r-universe.dev/) -[![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.6602574.svg)](https://doi.org/10.5281/zenodo.6602574) +[![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.6602573.svg)](https://doi.org/10.5281/zenodo.6602573) R Package with Functions for Scraping Data of Wasserportal Berlin (https://wasserportal.berlin.de), which contains From 1fec8ba05077535a28be07c44e5eead7b6589d36 Mon Sep 17 00:00:00 2001 From: mrustl Date: Thu, 9 Jun 2022 09:52:26 +0200 Subject: [PATCH 12/12] Re-run also other vignettes that werre temporarilly deactivated --- vignettes/{.groundwater.Rmd => groundwater.Rmd} | 0 .../{.promisces_norman-lists.Rmd => promisces_norman-lists.Rmd} | 0 vignettes/{.promisces_reach-uba.Rmd => promisces_reach-uba.Rmd} | 0 vignettes/{.tutorial.Rmd => tutorial.Rmd} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename vignettes/{.groundwater.Rmd => groundwater.Rmd} (100%) rename vignettes/{.promisces_norman-lists.Rmd => promisces_norman-lists.Rmd} (100%) rename vignettes/{.promisces_reach-uba.Rmd => promisces_reach-uba.Rmd} (100%) rename vignettes/{.tutorial.Rmd => tutorial.Rmd} (100%) diff --git a/vignettes/.groundwater.Rmd b/vignettes/groundwater.Rmd similarity index 100% rename from vignettes/.groundwater.Rmd rename to vignettes/groundwater.Rmd diff --git a/vignettes/.promisces_norman-lists.Rmd b/vignettes/promisces_norman-lists.Rmd similarity index 100% rename from vignettes/.promisces_norman-lists.Rmd rename to vignettes/promisces_norman-lists.Rmd diff --git a/vignettes/.promisces_reach-uba.Rmd b/vignettes/promisces_reach-uba.Rmd similarity index 100% rename from vignettes/.promisces_reach-uba.Rmd rename to vignettes/promisces_reach-uba.Rmd diff --git a/vignettes/.tutorial.Rmd b/vignettes/tutorial.Rmd similarity index 100% rename from vignettes/.tutorial.Rmd rename to vignettes/tutorial.Rmd