v2.0.5.9012

ggplot in QC report, issue #83, issue #84

update file import to na.strings to inclue "NA" not just "".

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: ContDataQC
 Title: Quality Control (QC) of Continous Monitoring Data
-Version: 2.0.5.9011
+Version: 2.0.5.9012
 Authors@R: c(
 	person("Erik W.", "Leppo", email="[email protected]",role=c("aut","cre")),
 	person("Ann","Roseberry Lincoln", role="ctb"),
@@ -17,13 +17,13 @@ BugReports: https://github.com/leppott/ContDataQC/issues
 Imports:
     dataRetrieval,
     doBy,
+    ggplot2,
     knitr,
     rmarkdown,
     shiny,    
     survival,
     zoo
 Suggests:
-    ggplot2,
     IHA,
     installr,
     rLakeAnalyzer,

NEWS

@@ -3,7 +3,7 @@ NEWS-ContDataQC
 
 <!-- NEWS.md is generated from NEWS.Rmd. Please edit that file -->
 
-    #> Last Update: 2019-02-19 13:24:28
+    #> Last Update: 2019-02-21 10:13:18
 
 # Planned Updates
 
@@ -35,6 +35,14 @@ NEWS-ContDataQC
 
 # Version History
 
+## v2.0.5.9012
+
+2019-02-21
+
+  - Update plots in QC Report to ggplot. Issues \#83 and 84.
+  - Import files fix, na.strings=
+      - Use c(“”,“NA”) not just “”.
+
 ## v2.0.5.9011
 
 2019-02-19

NEWS.rmd

@@ -44,6 +44,13 @@ cat(paste0("Last Update: ",Sys.time()))
 
 # Version History
 
+## v2.0.5.9012
+2019-02-21
+
+* Update plots in QC Report to ggplot.  Issues #83 and 84.
+* Import files fix, na.strings=
+    + Use c("","NA") not just "".
+
 ## v2.0.5.9011
 2019-02-19
 

R/fun.PeriodStats.R

@@ -228,7 +228,7 @@ PeriodStats <- function(fun.myDate
     #
   }##IF.file.END
   # 2.2. Load File
-  df.load <- read.csv(file.path(fun.myDir.import, fun.myFile),as.is=TRUE,na.strings="")
+  df.load <- read.csv(file.path(fun.myDir.import, fun.myFile),as.is=TRUE,na.strings=c("","NA"))
   # 2.3. Error Checking, data field names
   param.len <- length(fun.myParam.Name)
   myNames2Match <- c(fun.myParam.Name, fun.myDateTime.Name)

R/fun.QC.File.R

@@ -244,8 +244,8 @@ fun.QC.File <- function(fun.myFile
     #data.import=read.table(strFile,header=F,varSep)
     #varSep = "\t" (use read.delim instead of read.table)
     # as.is = T so dates come in as text rather than factor
-    #data.import <- read.delim(strFile,as.is=TRUE,na.strings="")
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    #data.import <- read.delim(strFile,as.is=TRUE,na.strings=c("","NA"))
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
     #
     # QC required fields: SiteID & (DateTime | (Date & Time))
     fun.QC.ReqFlds(names(data.import),file.path(myDir.data.import,strFile))
@@ -806,7 +806,7 @@ fun.QC.File <- function(fun.myFile
     #************************
 
 
-    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
+    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
     #
     # B.4.0. Columns
     # B.4.1. Check for DataFields  (may have already been done)

R/fun.QC.R

@@ -60,6 +60,8 @@ fun.QC <- function(fun.myData.SiteID
                    , fun.myReport.Dir
                    , fun.CreateReport=TRUE) {##FUN.fun.QC.START
   #
+  boo_DEBUG <- "FALSE"
+
   # A. Data Prep ####
   # Convert Data Type to proper case
   fun.myData.Type <- paste(toupper(substring(fun.myData.Type,1,1)),tolower(substring(fun.myData.Type,2,nchar(fun.myData.Type))),sep="")
@@ -207,8 +209,8 @@ fun.QC <- function(fun.myData.SiteID
     #varSep = "\t" (use read.delim instead of read.table)
     # as.is = T so dates come in as text rather than factor
     #data.import <- read.delim(strFile,as.is=TRUE,na.strings="")
-   # data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+   # data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
     #
     # QC required fields: SiteID & (DateTime | (Date & Time))
     fun.QC.ReqFlds(names(data.import),paste(myDir.data.import,strFile,sep="/"))
@@ -794,7 +796,7 @@ fun.QC <- function(fun.myData.SiteID
     #************************
 
 
-    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
+    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
     #
     # B.4.0. Columns
     # B.4.1. Check for DataFields  (may have already been done)
@@ -860,6 +862,18 @@ fun.QC <- function(fun.myData.SiteID
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     # insert QC Report so runs without user intervention
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # DEBUG, REPORT ####
+    if(boo_DEBUG==TRUE){##IF~boo_DEBUG~START
+      fun.myData.SiteID           <- strFile.SiteID
+      fun.myData.Type             <- strFile.DataType
+      fun.myData.DateRange.Start  <- fun.myData.DateRange.Start
+      fun.myData.DateRange.End    <- fun.myData.DateRange.End
+      fun.myDir.BASE              <- fun.myDir.BASE
+      fun.myDir.SUB.import        <- fun.myDir.SUB.export
+      fun.myDir.SUB.export        <- fun.myDir.SUB.export
+      fun.myFile.Prefix           <- strFile.Out.Prefix
+    }##IF~boo_DEBUG~END
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     # run with same import and export directory
     ###
     # B.10.3. Report ####
@@ -876,19 +890,6 @@ fun.QC <- function(fun.myData.SiteID
                  ) }##IF.CreateReport.END
 
     #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# QC
-# ##
-#                     fun.myData.SiteID           <- strFile.SiteID
-#                     fun.myData.Type             <- strFile.DataType
-#                     fun.myData.DateRange.Start  <- fun.myData.DateRange.Start
-#                     fun.myData.DateRange.End    <- fun.myData.DateRange.End
-#                     fun.myDir.BASE              <- fun.myDir.BASE
-#                     fun.myDir.SUB.import        <- fun.myDir.SUB.export
-#                     fun.myDir.SUB.export        <- fun.myDir.SUB.export
-#                     fun.myFile.Prefix           <- strFile.Out.Prefix
-    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     # B.11. Clean up
     # B.11.1. Inform user of progress and update LOG

R/fun.Report.File.R

@@ -124,7 +124,7 @@ fun.Report.File <- function(fun.myFile
     }
 
     #import the file
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
 
 
     # pick 'report' based on prefix

R/fun.Report.R

@@ -109,8 +109,8 @@ fun.Report <- function(fun.myData.SiteID
     }##IF.file.END
 
     #import the file
-    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
 
 
     # pick 'report' based on prefix

R/fun.Stats.File.R

@@ -174,7 +174,7 @@ fun.Stats.File <- function(fun.myFile
 
 
     #import the file
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
     #
     # QC required fields: SiteID & (DateTime | (Date & Time))
     #fun.QC.ReqFlds(names(data.import),paste(myDir.data.import,strFile,sep="/"))

R/fun.Stats.R

@@ -140,8 +140,8 @@ fun.Stats <- function(fun.myData.SiteID
   }##IF.file.END
 
   #import the file
-  #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
-  data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+  #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+  data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
   #
   # QC required fields: SiteID & (DateTime | (Date & Time))
   #fun.QC.ReqFlds(names(data.import),paste(myDir.data.import,strFile,sep="/"))

R/zfun.AggregateData.File.R

@@ -267,8 +267,8 @@ fun.AggregateData.File <- function(fun.myFile
     #data.import=read.table(strFile,header=F,varSep)
     #varSep = "\t" (use read.delim instead of read.table)
     # as.is = T so dates come in as text rather than factor
-    #data.import <- read.delim(strFile,as.is=TRUE,na.strings="")
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    #data.import <- read.delim(strFile,as.is=TRUE,na.strings=c("","NA"))
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
     #
     # QC required fields: SiteID & (DateTime | (Date & Time))
     fun.QC.ReqFlds(names(data.import),file.path(myDir.data.import,strFile))

R/zfun.AggregateData.R

@@ -228,9 +228,9 @@ fun.AggregateData <- function(fun.myData.SiteID
     #data.import=read.table(strFile,header=F,varSep)
     #varSep = "\t" (use read.delim instead of read.table)
     # as.is = T so dates come in as text rather than factor
-    #data.import <- read.delim(strFile,as.is=TRUE,na.strings="")
-    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings="")
-    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings="")
+    #data.import <- read.delim(strFile,as.is=TRUE,na.strings=c("","NA"))
+    #data.import <- read.csv(paste(myDir.data.import,strFile,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+    data.import <- read.csv(file.path(myDir.data.import,strFile),as.is=TRUE,na.strings=c("","NA"))
     #
     # QC required fields: SiteID & (DateTime | (Date & Time))
     #fun.QC.ReqFlds(names(data.import),paste(myDir.data.import,strFile,sep="/"))
@@ -438,10 +438,10 @@ fun.AggregateData <- function(fun.myData.SiteID
     #     fun.Name.myDF.2    <- myDF.Name.2
     #
     # Load Files
-    #data.DF.1 <- read.csv(paste(myDir.data.export,fun.Name.myDF.1,sep="/"),as.is=TRUE,na.strings="")
-    #data.DF.2 <- read.csv(paste(myDir.data.export,fun.Name.myDF.2,sep="/"),as.is=TRUE,na.strings="")
-    data.DF.1 <- read.csv(file.path(myDir.data.export,fun.Name.myDF.1),as.is=TRUE,na.strings="")
-    data.DF.2 <- read.csv(file.path(myDir.data.export,fun.Name.myDF.2),as.is=TRUE,na.strings="")
+    #data.DF.1 <- read.csv(paste(myDir.data.export,fun.Name.myDF.1,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+    #data.DF.2 <- read.csv(paste(myDir.data.export,fun.Name.myDF.2,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+    data.DF.1 <- read.csv(file.path(myDir.data.export,fun.Name.myDF.1),as.is=TRUE,na.strings=c("","NA"))
+    data.DF.2 <- read.csv(file.path(myDir.data.export,fun.Name.myDF.2),as.is=TRUE,na.strings=c("","NA"))
     #
     # strip non file specific columns ????
     # drop overlapping field names in data.DF.2 (typically the gage file) but keep DateTime
@@ -759,8 +759,8 @@ fun.AggregateData <- function(fun.myData.SiteID
 #
 #
 #       # Load Files
-#       data.air <- read.csv(paste(myDir.data.export,Name.File.Air,sep="/"),as.is=TRUE,na.strings="")
-#       data.water <- read.csv(paste(myDir.data.export,Name.File.Water,sep="/"),as.is=TRUE,na.strings="")
+#       data.air <- read.csv(paste(myDir.data.export,Name.File.Air,sep="/"),as.is=TRUE,na.strings=c("","NA"))
+#       data.water <- read.csv(paste(myDir.data.export,Name.File.Water,sep="/"),as.is=TRUE,na.strings=c("","NA"))
 #       # strip non-file specific columns
 #       myNames.Order.Air <- c(myName.SiteID,myName.Date,myName.Time,myName.DateTime,myName.AirTemp,myName.LoggerID.Air,myName.RowID.Air)
 #       myNames.Order.Water <-c(myName.DateTime,myName.WaterTemp,myName.WaterP,myName.AirBP,myName.SensorDepth,myName.LoggerID.Water,myName.RowID.Water)
@@ -822,7 +822,7 @@ fun.AggregateData <- function(fun.myData.SiteID
 #       #
 #     } else if (files.match[3]==TRUE) {
 #       #
-#       data.AW <- read.csv(paste(myDir.data.export,Name.File.AW,sep="/"),as.is=TRUE,na.strings="")
+#       data.AW <- read.csv(paste(myDir.data.export,Name.File.AW,sep="/"),as.is=TRUE,na.strings=c("","NA"))
 #       #
 #     }##IF.air/water.END
 #
@@ -904,7 +904,7 @@ fun.AggregateData <- function(fun.myData.SiteID
 #   # Air
 #   if(files.match[1]==TRUE){##IF.filesmatch_air.START
 #     #
-#     data.air <- read.csv(paste(myDir.data.export,Name.File.Air,sep="/"),as.is=TRUE,na.strings="")
+#     data.air <- read.csv(paste(myDir.data.export,Name.File.Air,sep="/"),as.is=TRUE,na.strings=c("","NA"))
 #     data.plot <- data.air
 #     # cheat on Date/Time axis
 #     n.Total <- length(data.plot[,myName.Date])
@@ -927,7 +927,7 @@ fun.AggregateData <- function(fun.myData.SiteID
 #   # Water
 #   if(files.match[2]==TRUE){##IF.filesmatch_water.START
 #     #
-#     data.water <- read.csv(paste(myDir.data.export,Name.File.Water,sep="/"),as.is=TRUE,na.strings="")
+#     data.water <- read.csv(paste(myDir.data.export,Name.File.Water,sep="/"),as.is=TRUE,na.strings=c("","NA"))
 #     data.plot <- data.water
 #     # cheat on Date/Time axis
 #     n.Total <- length(data.plot[,myName.Date])

R/zfun.ContDataQC.R

@@ -276,6 +276,25 @@ ContDataQC <- function(fun.myData.Operation
                        , fun.myReport.format=""
                        , fun.myReport.Dir=""
                        , fun.CreateReport=TRUE) {##FUN.fun.Master.START
+  # DEBUG ####
+  boo_DEBUG <- FALSE
+  if(boo_DEBUG==TRUE){##IF~boo_DEBUG~START
+    fun.myData.Operation       <- myData.Operation
+    fun.myData.SiteID          <- myData.SiteID
+    fun.myData.Type            <- myData.Type
+    fun.myData.DateRange.Start <- myData.DateRange.Start
+    fun.myData.DateRange.End   <- myData.DateRange.End
+    fun.myDir.import           <- myDir.import
+    fun.myDir.export           <- myDir.export
+    fun.myConfig               <- ""
+    #fun.myFile                 <- ""
+    fun.myFile                 <- myFile
+    fun.myReport.format        <- myReport.format
+    fun.myReport.Dir           <- ""
+    source(file.path(getwd(), "inst", "extdata", "config.ORIG.R"))
+    source(file.path(getwd(), "R", "fun.Helper.R"))
+  }##IF~boo_DEBUG~END
+
   # config file load, 20170517
   if (fun.myConfig!="") {##IF.fun.myConfig.START
     config.load(fun.myConfig)