refactor: version 1.0.7 final changes

MazamaScience · Feb 4, 2021 · f446cbc · f446cbc
1 parent 8b0530f
commit f446cbc
Show file tree

Hide file tree

Showing 165 changed files with 528 additions and 222 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Type: Package
 Package: AirSensor
-Version: 1.0.6
+Version: 1.0.7
 Title: Process and Display Data from Air Quality Sensors
 Authors@R: c(
     person("Jonathan", "Callahan", email="[email protected]", role=c("aut","cre")), 

diff --git a/NAMESPACE b/NAMESPACE
@@ -7,7 +7,7 @@ export(PurpleAirQC_hourly_AB_02)
 export(PurpleAirQC_hourly_AB_03)
 export(PurpleAirSoH_dailyABFit)
 export(PurpleAirSoH_dailyABtTest)
-export(PurpleAirSoH_dailyOtherFit)
+export(PurpleAirSoH_dailyMetFit)
 export(PurpleAirSoH_dailyPctDC)
 export(PurpleAirSoH_dailyPctReporting)
 export(PurpleAirSoH_dailyPctValid)

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,14 @@
+# AirSensor 1.0.7
+
+* Minor documentation updates to clarify "state-of-health" calculations.
+* Improved pattern matching for South Coast communities in `pas_addCommunityRegion()`.
+* Remove "Voc" column during `pas_load()` so that current and archival 'pas' 
+objects can be easily merged. (Some early 'pas' objects contained some "Voc"
+data but this has never been a focus of the **AirSensor** package. Later 'pas'
+objects do not contain the "Voc" parameter.)
+* `pas_load()` now properly handles loading of archival 'pas' objects prior to
+2019-08-02.
+
 # AirSensor 1.0.6
 
 * Fixed documentation and included dataset encoding to UTF-8 standard.
@@ -450,7 +461,7 @@ labels (currently used as unique identifiers)
 Refactored `PurpleAirSoH_daily~()` functions to use *dplyr* resulting in code
 that runs faster and is easier to understand.
 
-* Changed `~SoH_dailyCorrelation()` to `~SoH_dailyOtherFit()`.
+* Changed `~SoH_dailyCorrelation()` to `~SoH_dailyMetFit()`.
 * Changed `timeseriesTbl_multiPlot()` argument `parameterPatter` to `pattern`.
 
 # AirSensor 0.5.11

diff --git a/R/PurpleAirSoH_dailyABtTest.R b/R/PurpleAirSoH_dailyABtTest.R
@@ -9,8 +9,9 @@
 #' @description This function calculates a t-test between
 #' the \code{pm25_A}, \code{pm25_B}. A t-statistic and a p-value will be 
 #' returned for each day. All returned values are expected to hover near 0 for a 
-#' properly functioning sensor.
-#' 
+#' properly functioning sensor. The t-statistic and p-value serve to test whether 
+#' or not the \code{pm25_A} and \code{pm25_B} data are significantly different 
+#' based on a student’s t-test.
 #' 
 #' @examples  
 #' library(AirSensor)

diff --git a/R/PurpleAirSoH_dailyOtherFit.R → R/PurpleAirSoH_dailyMetFit.R b/R/PurpleAirSoH_dailyOtherFit.R → R/PurpleAirSoH_dailyMetFit.R
@@ -19,7 +19,7 @@
 #'  
 #' tbl <- 
 #'   example_pat_failure_A %>%
-#'   PurpleAirSoH_dailyOtherFit() 
+#'   PurpleAirSoH_dailyMetFit() 
 #'   
 #' timeseriesTbl_multiPlot(
 #'   tbl, 
@@ -33,7 +33,7 @@
 #' )
 #' 
 
-PurpleAirSoH_dailyOtherFit <- function(
+PurpleAirSoH_dailyMetFit <- function(
   pat = NULL
 ) {
 
@@ -77,7 +77,7 @@ PurpleAirSoH_dailyOtherFit <- function(
     datetime = MazamaCoreUtils::dateSequence(start, end - lubridate::ddays(1), timezone = timezone)
   )
 
-  # ----- Calculate dailyOtherFit -------------------------------------------
+  # ----- Calculate dailyMetFit -------------------------------------------
 
   # Note: This function uses a combination of lm() and summary() to calculate 
   # the r-squared between several channels rather than using stats::cor() 
@@ -91,7 +91,7 @@ PurpleAirSoH_dailyOtherFit <- function(
   #                      enddate = "2019-09-11",
   #                      timezone = "America/Los_Angeles")
   # cor_stats <- (stats::cor(pat$data$pm25_A, pat$data$temperature, use = "pairwise.complete.obs"))^2
-  # lm_rsquared <- PurpleAirSoH_dailyOtherFit(pat)
+  # lm_rsquared <- PurpleAirSoH_dailyMetFit(pat)
   # Notice that cor_stats and lm_rsquared$pm25_A_temperature_rsquared are the same.
 
 

diff --git a/R/PurpleAirSoH_dailyPctDC.R b/R/PurpleAirSoH_dailyPctDC.R
@@ -13,6 +13,10 @@
 #' The number of hours with a DC signal are summed over the day and a daily DC 
 #' percentage for each channel is returned. 
 #' 
+#' This metric allows users to identify “sticky values”, or instances of a sensor 
+#' continuously logging the same value. A high percent DC value indicates the 
+#' likely occurrence of a “sticky value”, and a zero or low percent DC indicates 
+#' that the sensor is recording dynamic data.
 #' 
 #' @examples  
 #' library(AirSensor)

diff --git a/R/PurpleAirSoH_dailyPctValid.R b/R/PurpleAirSoH_dailyPctValid.R
@@ -9,8 +9,9 @@
 #' @description The number of valid (\emph{i.e.}, not NA or out-of-spec) sensor 
 #' measurements are summed over the course of a calendar day, then divided by the 
 #' total number of measurements the sensor actually recorded during that day 
-#' (including NA and out-of-spec values) to return a percentage of the total
-#' recorded measurements that are considered plausible.
+#' (including NA and out-of-spec values) to return a percentage of the total 
+#' recorded measurements that are considered plausible. This metric utilizes the 
+#' same bounds as the \code{pat_qc()} function to identify out-of-spec values.
 #' 
 #' @examples
 #' library(AirSensor)

diff --git a/R/pas_addCommunityRegion.R b/R/pas_addCommunityRegion.R
@@ -49,22 +49,22 @@ pas_addCommunityRegion <- function(
   #  ----- SCAQMD communities --------------------------------------------------
 
   # NOTE:  Need to match "sctv_15 (dawson canyon) b"
-  scah_mask <- stringr::str_detect(label, "^scah_[0-9][0-9]( ?.*$)")
-  scan_mask <- stringr::str_detect(label, "^scan_[0-9][0-9]( ?.*$)")
-  scap_mask <- stringr::str_detect(label, "^scap_[0-9][0-9]( ?.*$)")
-  scbb_mask <- stringr::str_detect(label, "^scbb_[0-9][0-9]( ?.*$)")
-  scem_mask <- stringr::str_detect(label, "^scem_[0-9][0-9]( ?.*$)")
-  schs_mask <- stringr::str_detect(label, "^schs_[0-9][0-9]( ?.*$)")
-  sciv_mask <- stringr::str_detect(label, "^sciv_[0-9][0-9]( ?.*$)")
-  scnp_mask <- stringr::str_detect(label, "^scnp_[0-9][0-9]( ?.*$)")
-  scpr_mask <- stringr::str_detect(label, "^scpr_[0-9][0-9]( ?.*$)")
-  scsb_mask <- stringr::str_detect(label, "^scsb_[0-9][0-9]( ?.*$)")
-  scsc_mask <- stringr::str_detect(label, "^scsc_[0-9][0-9]( ?.*$)")
-  scsg_mask <- stringr::str_detect(label, "^scsg_[0-9][0-9]( ?.*$)")
-  scsh_mask <- stringr::str_detect(label, "^scsh_[0-9][0-9]( ?.*$)")
-  scsj_mask <- stringr::str_detect(label, "^scsj_[0-9][0-9]( ?.*$)")
-  sctv_mask <- stringr::str_detect(label, "^sctv_[0-9][0-9]( ?.*$)")
-  scuv_mask <- stringr::str_detect(label, "^scuv_[0-9][0-9]( ?.*$)")
+  scah_mask <- stringr::str_detect(label, "^scah_[0-9]{1,2}( ?.*$)")
+  scan_mask <- stringr::str_detect(label, "^scan_[0-9]{1,2}( ?.*$)")
+  scap_mask <- stringr::str_detect(label, "^scap_[0-9]{1,2}( ?.*$)")
+  scbb_mask <- stringr::str_detect(label, "^scbb_[0-9]{1,2}( ?.*$)")
+  scem_mask <- stringr::str_detect(label, "^scem_[0-9]{1,2}( ?.*$)")
+  schs_mask <- stringr::str_detect(label, "^schs_[0-9]{1,2}( ?.*$)")
+  sciv_mask <- stringr::str_detect(label, "^sciv_[0-9]{1,2}( ?.*$)")
+  scnp_mask <- stringr::str_detect(label, "^scnp_[0-9]{1,2}( ?.*$)")
+  scpr_mask <- stringr::str_detect(label, "^scpr_[0-9]{1,2}( ?.*$)")
+  scsb_mask <- stringr::str_detect(label, "^scsb_[0-9]{1,2}( ?.*$)")
+  scsc_mask <- stringr::str_detect(label, "^scsc_[0-9]{1,2}( ?.*$)")
+  scsg_mask <- stringr::str_detect(label, "^scsg_[0-9]{1,2}( ?.*$)")
+  scsh_mask <- stringr::str_detect(label, "^scsh_[0-9]{1,2}( ?.*$)")
+  scsj_mask <- stringr::str_detect(label, "^scsj_[0-9]{1,2}( ?.*$)")
+  sctv_mask <- stringr::str_detect(label, "^sctv_[0-9]{1,2}( ?.*$)")
+  scuv_mask <- stringr::str_detect(label, "^scuv_[0-9]{1,2}( ?.*$)")
 
   pas$communityRegion[scah_mask] <- "SCAH"
   pas$communityRegion[scan_mask] <- "SCAN"

diff --git a/R/pas_load.R b/R/pas_load.R
@@ -143,6 +143,11 @@ pas_load <- function(
       dataDir <- paste0(baseDir, '/pas/', yearstamp)
       dataUrl <- NULL
     }
+
+    # Edge Case: pas files before 20190802 do not have 'archival' at the end
+    if ( as.numeric(datestamp) < 20190802 ) {
+      filename <- stringr::str_replace(filename, '_archival', '')
+    }
 
     # TODO:  Revisit better ways to ignore loadDataFile() messages
 
@@ -184,6 +189,16 @@ pas_load <- function(
     }
   }
 
+  # ----- Harmonize data -------------------------------------------------------
+
+  # NOTE:  Some early 'pas' objects have a "Voc" column which is missing in later
+  # NOTE:  'pas' objects. This creates confusion when people attempt to merge
+  # NOTE:  current and archival 'pas' objects. We make an executive decision
+  # NOTE:  here to remove "Voc" as this is not the focus of this package.
+
+  if ( "Voc" %in% names(pas) ) {
+    pas$Voc <- NULL
+  }
 
   # ----- Return ---------------------------------------------------------------
 

diff --git a/R/pat_dailySoH.R b/R/pat_dailySoH.R
@@ -30,7 +30,7 @@ pat_dailySoH <- function(
   SoH_functions = c("PurpleAirSoH_dailyPctDC", 
                     "PurpleAirSoH_dailyPctReporting", 
                     "PurpleAirSoH_dailyPctValid", 
-                    "PurpleAirSoH_dailyOtherFit", 
+                    "PurpleAirSoH_dailyMetFit", 
                     "PurpleAirSoH_dailyABFit",
                     "PurpleAirSoH_dailyABtTest")
 ) {
@@ -107,7 +107,7 @@ if ( FALSE ) {
   SoH_functions = c("PurpleAirSoH_dailyPctDC", 
                     "PurpleAirSoH_dailyPctReporting", 
                     "PurpleAirSoH_dailyPctValid", 
-                    "PurpleAirSoH_dailyOtherFit", 
+                    "PurpleAirSoH_dailyMetFit", 
                     "PurpleAirSoH_dailyABFit",
                     "PurpleAirSoH_dailyABtTest")
 

diff --git a/docker/AirSensor_1.0.6.tar.gz b/docker/AirSensor_1.0.6.tar.gz
diff --git a/docker/AirSensor_1.0.7.tar.gz b/docker/AirSensor_1.0.7.tar.gz
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -32,7 +32,7 @@ RUN wget -nv http://mazamascience.com/RData/Spatial/CA_AirBasins_01.RData \
 
 WORKDIR /
 
-COPY AirSensor_1.0.6.tar.gz /
+COPY AirSensor_1.0.7.tar.gz /
 
-RUN R CMD INSTALL AirSensor_1.0.6.tar.gz
+RUN R CMD INSTALL AirSensor_1.0.7.tar.gz
 
diff --git a/docker/Makefile b/docker/Makefile
@@ -3,14 +3,14 @@
 
 AirSensor_tarball:
 	cd ..; R CMD build --resave-data .
-	mv ../AirSensor_1.0.6.tar.gz .
+	mv ../AirSensor_1.0.7.tar.gz .
 
 # PRODUCTION version -----------------------------------------------------------
 
 production_build:
-	docker build --no-cache -t mazamascience/airsensor:1.0.6 -t mazamascience/airsensor:latest .
+	docker build --no-cache -t mazamascience/airsensor:1.0.7 -t mazamascience/airsensor:latest .
 
 production_publish:
-	docker login && docker push mazamascience/airsensor:1.0.6
+	docker login && docker push mazamascience/airsensor:1.0.7
 
 
diff --git a/docker/README.md b/docker/README.md
@@ -16,7 +16,7 @@ You should then be able to see something like the following:
 $ docker images
 REPOSITORY                    TAG         IMAGE ID            CREATED             SIZE
 ...
-mazamascience/airsensor       1.0.6       796ae3ba57e5        2 minutes ago       3.74GB
+mazamascience/airsensor       1.0.7       796ae3ba57e5        2 minutes ago       3.74GB
 mazamascience/airsensor       latest      796ae3ba57e5        2 minutes ago       3.74GB
 ...
 ```
@@ -87,6 +87,6 @@ make production_publish
 A recent image can also be obtained from DockerHub with:
 
 ```
-docker pull mazamascience/airsensor:1.0.6
+docker pull mazamascience/airsensor:1.0.7
 ```
 
diff --git a/docs/404.html b/docs/404.html
diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html
diff --git a/docs/articles/AirSensor_Function_Overview.html b/docs/articles/AirSensor_Function_Overview.html
diff --git a/docs/articles/Developer_Style_Guide.html b/docs/articles/Developer_Style_Guide.html
diff --git a/docs/articles/articles/Australia_on_fire.html b/docs/articles/articles/Australia_on_fire.html
diff --git a/docs/articles/articles/Australia_on_fire_files/figure-html/view_pats-1.png b/docs/articles/articles/Australia_on_fire_files/figure-html/view_pats-1.png
diff --git a/docs/articles/articles/Custom_QC_Algorithms.html b/docs/articles/articles/Custom_QC_Algorithms.html
diff --git a/docs/articles/articles/DataFlowChart.html b/docs/articles/articles/DataFlowChart.html
diff --git a/docs/articles/articles/SoH_functions.html b/docs/articles/articles/SoH_functions.html
diff --git a/docs/articles/articles/Temporal_Aggregation.html b/docs/articles/articles/Temporal_Aggregation.html
diff --git a/docs/articles/articles/outlier_detection.html b/docs/articles/articles/outlier_detection.html
diff --git a/docs/articles/articles/pas_introduction.html b/docs/articles/articles/pas_introduction.html
diff --git a/docs/articles/articles/pat_introduction.html b/docs/articles/articles/pat_introduction.html
diff --git a/docs/articles/articles/pat_introduction_files/figure-html/allPlot-1.png b/docs/articles/articles/pat_introduction_files/figure-html/allPlot-1.png
diff --git a/docs/articles/articles/pat_introduction_files/figure-html/augustPlot-1.png b/docs/articles/articles/pat_introduction_files/figure-html/augustPlot-1.png
diff --git a/docs/articles/articles/pat_introduction_files/figure-html/scatterplot-1.png b/docs/articles/articles/pat_introduction_files/figure-html/scatterplot-1.png
diff --git a/docs/articles/articles/purpleair_failure_modes.html b/docs/articles/articles/purpleair_failure_modes.html
diff --git a/...ticles/purpleair_failure_modes_files/figure-html/channel_a_humidity_multi-1.png b/...ticles/purpleair_failure_modes_files/figure-html/channel_a_humidity_multi-1.png
diff --git a/...cles/purpleair_failure_modes_files/figure-html/channel_a_humidity_scatter-1.png b/...cles/purpleair_failure_modes_files/figure-html/channel_a_humidity_scatter-1.png
diff --git a/...rticles/articles/purpleair_failure_modes_files/figure-html/channel_a_jump-1.png b/...rticles/articles/purpleair_failure_modes_files/figure-html/channel_a_jump-1.png
diff --git a/...cles/articles/purpleair_failure_modes_files/figure-html/channel_a_noise_2-1.png b/...cles/articles/purpleair_failure_modes_files/figure-html/channel_a_noise_2-1.png
diff --git a/...ticles/articles/purpleair_failure_modes_files/figure-html/unnamed-chunk-1-1.png b/...ticles/articles/purpleair_failure_modes_files/figure-html/unnamed-chunk-1-1.png