Merge pull request #20 from KWB-R/dev

Add PROMISCES Workflow for GW analysis

.github/workflows/R-CMD-check.yaml

@@ -32,6 +32,7 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
+      ZENODO_PAT: ${{ secrets.ZENODO_PAT }}
 
     steps:
       - uses: actions/checkout@v2
@@ -71,7 +72,7 @@ jobs:
         shell: Rscript {0}
 
       - name: Check
-        run: rcmdcheck::rcmdcheck(args = "--no-manual", error_on = "warning", check_dir = "check")
+        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--ignore-vignettes", "--no-build-vignettes"), build_args = c("--no-manual", "--ignore-vignettes", "--no-build-vignettes"), error_on = "error", check_dir = "check")
         shell: Rscript {0}
 
       - name: Upload check results

.github/workflows/pkgdown.yaml

@@ -15,6 +15,7 @@ jobs:
     runs-on: macOS-latest
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      ZENODO_PAT: ${{ secrets.ZENODO_PAT }}
     steps:
       - uses: actions/checkout@v2
 

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: wasserportal
 Title: R Package with Functions for Scraping Data of
     Wasserportal Berlin
-Version: 0.0.0.9000
+Version: 0.1.0
 Authors@R: 
     c(person(given = "Hauke",
              family = "Sonnenberg",
@@ -15,6 +15,8 @@ Authors@R:
              comment = c(ORCID = "0000-0003-0647-7726")),
       person(given = "DWC",
              role = "fnd"),
+      person(given = "PROMISCES",
+             role = "fnd"),
       person(given = "Kompetenzzentrum Wasser Berlin gGmbH (KWB)",
              role = "cph"))
 Description: R Package with Functions for Scraping Data of
@@ -39,22 +41,27 @@ Suggests:
     covr,
     DT,
     forcats,
+    fs,
     htmlwidgets,
+    janitor,
     jsonlite,
     leaflet,
     ggplot2,
+    gridExtra,
     knitr,
     kwb.pkgbuild,
     openxlsx,
     plotly,
+    readr,
     rmarkdown,
-    sf
+    sf,
+    tidyselect
 Remotes: 
     github::kwb-r/kwb.datetime,
     github::kwb-r/kwb.pkgbuild,
     github::kwb-r/kwb.utils
 Encoding: UTF-8
 LazyData: true
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.1.1
+RoxygenNote: 7.1.2
 VignetteBuilder: knitr

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Kompetenzzentrum Wasser Berlin gGmbH (KWB)
+Copyright (c) 2021-2022 Kompetenzzentrum Wasser Berlin gGmbH (KWB)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

LICENSE.md

@@ -1,6 +1,6 @@
 # MIT License
 
-Copyright (c) 2021 Kompetenzzentrum Wasser Berlin gGmbH (KWB)
+Copyright (c) 2021-2022 Kompetenzzentrum Wasser Berlin gGmbH (KWB)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

NAMESPACE

@@ -25,6 +25,8 @@ importFrom(dplyr,select)
 importFrom(httr,POST)
 importFrom(httr,content)
 importFrom(kwb.datetime,textToEuropeBerlinPosix)
+importFrom(kwb.utils,readPackageFile)
+importFrom(kwb.utils,selectColumns)
 importFrom(kwb.utils,substSpecialChars)
 importFrom(magrittr,"%>%")
 importFrom(parallel,detectCores)

NEWS.md

@@ -1,3 +1,34 @@
+# [wasserportal 0.1.0](https://github.com/KWB-R/wasserportal/releases/tag/v0.1.0) <small>2022-06-01</small>
+
+R package for scraping `groundwater` data (`groundwater level` and `quality`) from [Wasserportal Berlin](https://wasserportal.berlin.de). Please note that the 
+support for scraping `surface water` monitoring stations is currently very limited!
+
+**Functions:**
+
+* `get_stations()`: returns metadata for all available monitoring stations
+* `get_wasserportal_masters_data()`: get master data for selected `station_ids` 
+* `read_wasserportal_raw_gw()`: enables the download of `groundwater data`. 
+Checkout the [Tutorial](../articles/tutorial.html) article how to use it for downloading [one](../articles/tutorial.html#download-and-plotting-one-station) or [multiple](../articles/tutorial.html#download-and-plotting-multiple-stations) 
+stations at once.
+* `read_wasserportal()`: works for `surface water` monitoring stations, but is 
+outdated, as it is based on an outdated static file with station/variable names 
+(i.e. only for `11` instead of `82` stations currently provided!) instead of 
+relying on new metadata provided online. This will be fixed within the next release. For progress on this issue checkout [#21](https://github.com/KWB-R/wasserportal/issues/21)
+
+**Workflows:**
+
+- [Tutorial](../articles/tutorial.html) article how to download groundwater level 
+and quality data
+
+
+- **Further Usage** by combining previously scraped (see 
+[tutorial](../articles/tutorial.html) above) data and performing some analysis: 
+
+  * [Groundwater](../articles/groundwater.html), e.g. creating a map with GW level 
+  trends
+
+  * Two workflows ([REACH UBA](../articles/promisces_reach-uba.html), [Norman List](../articles/promisces_norman-lists.html)) created within the project [PROMISCES](https://www.kompetenz-wasser.de/en/forschung/projekte/promisces) for assessing  prevalence and the spatial distribution of **p**ersistent, **m**obile and **t**oxic (PMT) substances in the Berlin groundwater, based on different PMT lists, i.e. [REACH UBA](../articles/promisces_reach-uba.html) or [Norman List](../articles/promisces_norman-lists.html).
+
 # wasserportal 0.0.0.9000
 
 * Added a `NEWS.md` file to track changes to the package.

R/get_stations.R

@@ -10,6 +10,7 @@
 #' @importFrom dplyr select mutate left_join
 #' @importFrom tidyr pivot_wider
 #' @importFrom data.table rbindlist
+#' @importFrom rlang .data
 
 get_stations <- function(run_parallel = TRUE) {
 
@@ -50,7 +51,7 @@ overview_df <- data.table::rbindlist(overview_list, fill = TRUE, idcol = "key")
 metadata <- tidyr::separate(
   data.frame(key = names(overview_options),
              station_type = as.vector(overview_options)),
-  key,
+  .data$key,
   into = c("water_body", "variable"),
   sep = "\\.",
   remove = FALSE)

R/read_wasserportal.R

@@ -34,7 +34,7 @@
 #' variables <- wasserportal::get_wasserportal_variables()
 #'
 #' # Set the start date
-#' from_date <- "2020-03-01"
+#' from_date <- "2021-03-01"
 #'
 #' # Read the timeseries (multiple variables for one station)
 #' water_quality <- wasserportal::read_wasserportal(

R/read_wasserportal_raw_gw.R

@@ -52,7 +52,7 @@ read_wasserportal_raw_gw <- function(
   # Format the start date
   #
   if (from_date != "") sdatum <- format(from_date, format = "%d.%m.%Y")
-  if (sreihe == "wa") sdatum <- format(as.Date("1900-01-01"), format = "%d.%m.%Y")
+  if (sreihe == "wa") sdatum <- "01.01.1900"
 
   # Compose the body of the request
   body <- list(sreihe = sreihe,
@@ -105,11 +105,11 @@ read_wasserportal_raw_gw <- function(
                           names_to = c("parameter_unit"),
                           values_to = "Messwert") %>%
       dplyr::mutate(Messstellennummer = station,
-                    Parameter = stringr::str_remove(parameter_unit, "\\s+\\[.*\\]"),
-                    Einheit = stringr::str_extract(parameter_unit, "\\[.*\\]") %>%
+                    Parameter = stringr::str_remove(.data$parameter_unit, "\\s+\\[.*\\]"),
+                    Einheit = stringr::str_extract(.data$parameter_unit, "\\[.*\\]") %>%
                       stringr::str_remove(pattern = "\\[") %>%
                       stringr::str_remove(pattern = "\\]")) %>%
-      dplyr::filter(!is.na(Messwert)) %>%
+      dplyr::filter(!is.na(.data$Messwert)) %>%
       dplyr::select("Messstellennummer", "Datum", "Parameter", "Einheit", "Messwert")
   }
 
@@ -119,16 +119,16 @@ read_wasserportal_raw_gw <- function(
                           names_to = c("parameter_unit"),
                           values_to = "Messwert") %>%
       dplyr::mutate(Messstellennummer = station,
-                    Parameter = stringr::str_remove(parameter_unit, "\\s+\\(.*\\)"),
-                    Einheit = stringr::str_extract(parameter_unit, "\\(.*\\)") %>%
+                    Parameter = stringr::str_remove(.data$parameter_unit, "\\s+\\(.*\\)"),
+                    Einheit = stringr::str_extract(.data$parameter_unit, "\\(.*\\)") %>%
                       stringr::str_remove(pattern = "\\(") %>%
                       stringr::str_remove(pattern = "\\)")) %>%
-      dplyr::filter(!is.na(Messwert)) %>%
+      dplyr::filter(!is.na(.data$Messwert)) %>%
       dplyr::select("Messstellennummer", "Datum", "Parameter", "Einheit", "Messwert")
   }
 
   data <- data %>%
-    dplyr::mutate(Datum = as.Date(Datum, format = "%d.%m.%Y"))
+    dplyr::mutate(Datum = as.Date(.data$Datum, format = "%d.%m.%Y"))
 
   # Return the data frame with the additional fields of the header row as
   # meta information in attribute "metadata"

R/utils.R

@@ -45,6 +45,7 @@ read <- function(text, ...) {
 #' @param sep separator (default: ", ")
 #' @return vector of character with as many elements as there are rows in data
 #' @export
+#' @importFrom kwb.utils selectColumns
 #' @examples
 #' data <- data.frame(number = 1:2, name = c("adam", "eva"), value = 3:4)
 #' columns <- c("name", "value")
@@ -67,7 +68,7 @@ columns_to_labels <- function(data, columns, fmt = "%s: %s", sep = ", ")
 #' @return data frame representing the content of \code{\link{file}}
 #'
 #' @export
-#'
+#' @importFrom kwb.utils readPackageFile
 readPackageFile <- function(file, ...)
 {
   kwb.utils::readPackageFile(file, package = "wasserportal", ...)

_pkgdown.yml

@@ -1,19 +1,32 @@
+url: https://kwb-r.github.io/wasserportal
+
 authors:
   Hauke Sonnenberg:
     href: https://github.com/hsonne
   Michael Rustler:
     href: https://mrustl.de
   DWC:
-    href: https://www.kompetenz-wasser.de/en/project/dwc/
-    html: <img src='https://www.kompetenz-wasser.de/wp-content/uploads/2019/09/dwc-rvb-grey-516x244.jpg'
-      height='24' />
+    href: https://www.kompetenz-wasser.de/en/forschung/projekte/dwc
+    html: <img src='https://kompetenz-wasser.de/media/pages/forschung/projekte/dwc/6263c55e76-1639132760/dwc-rvb-grey.jpg'
+      alt='Project DWC' width='72' />
+  PROMISCES:
+    href: https://www.kompetenz-wasser.de/en/forschung/projekte/promisces
+    html: <img src='https://promisces.eu/skin/promisces/assets/img/logo.png'
+      alt='Project PROMISCES' width='72' />
   Kompetenzzentrum Wasser Berlin gGmbH (KWB):
-    href: http://www.kompetenz-wasser.de
-    html: <img src='http://www.kompetenz-wasser.de/wp-content/uploads/2017/08/cropped-logo-kwb_klein-new.png'
-      height='24' />
+    href: https://www.kompetenz-wasser.de
+    html: <img src='https://publications.kompetenz-wasser.de/img/KWB-Logo.svg'
+      alt='KWB' width='72' />
+
 template:
-  params:
-    bootswatch: cerulean
+  bootstrap: 5
+  bootswatch: cerulean
+  bslib:
+    bg: "#ffffff"
+    fg: "#000000"
+    primary: "#007aff"
+    border-radius: 0.5rem
+    btn-border-radius: 0.25rem
+
 development:
   mode: auto
-

index.md

@@ -11,20 +11,29 @@ real-time data of surface water and groundwater monitoring stations.
 
 ## Installation
 
-For details on how to install KWB-R packages checkout our [installation tutorial](https://kwb-r.github.io/kwb.pkgbuild/articles/install.html).
+For installing the latest release of this R package run the following code below:
 
 ```r
-### Optionally: specify GitHub Personal Access Token (GITHUB_PAT)
-### See here why this might be important for you:
-### https://kwb-r.github.io/kwb.pkgbuild/articles/install.html#set-your-github_pat
+# Enable repository from kwb-r
+options(repos = c(
+  kwbr = 'https://kwb-r.r-universe.dev',
+  CRAN = 'https://cloud.r-project.org'))
+
+# Download and install wasserportal in R
+install.packages('wasserportal')
+
+# Browse the wasserportal manual pages
+help(package = 'wasserportal')
+```
+## Usage 
+
+Checkout the [Tutorial](articles/tutorial.html) article on how to use this R package
+for scraping data from the [Wasserportal Berlin](https://wasserportal.berlin.de), 
+which is performed automatically every day at 5 am UTC using [GitHub actions](https://github.com/KWB-R/wasserportal/actions/workflows/pkgdown.yaml).
+
+How this data can be used directly from within R is shown in the [Groundwater](articles/groundwater.html) article.
+
+
 
-# Sys.setenv(GITHUB_PAT = "mysecret_access_token")
 
-# Install package "remotes" from CRAN
-if (! require("remotes")) {
-  install.packages("remotes", repos = "https://cloud.r-project.org")
-}
 
-# Install KWB package 'wasserportal' from GitHub
-remotes::install_github("KWB-R/wasserportal")
-```