Merge pull request #1017 from wadpac/issues_1002_1011_1016

Fixing various smaller issues

NEWS.md

@@ -1,5 +1,11 @@
 # CHANGES IN GGIR VERSION 3.0-4
 
+- Part 5: Improved handling of inconsistent number of columns in part 5 milestone data #1002
+
+- Part 3: Revised NotWorn option for argument HASPT.algo as used for count data #1011
+
+- Visualreport: Now able to handle recordings with no valid days argument visualreport_without_invalid = TRUE, fixes bug #1016
+
 - Part 2 and 5: Fix bug, timing of LX is now expressed on scale between 12 and 36 to allow for meaningful person level summary of this value #1012
 
 - Facilitate handling of Sensewear xls file format with externally derived epoch data #974

R/HASPT.R

@@ -50,11 +50,19 @@ HASPT = function(angle, perc = 10, spt_threshold = 15,
       }
     } else if (HASPT.algo == "NotWorn") {  
       # When protocol is to not wear sensor during the night,
-      # then look for longest period of zero or very low intensity
+      # and data is collected in count units we do not know angle
+      # as needed for HorAngle and HDCZA.
+      # Instead look for longest period of zero or very low intensity
+
+      # First attempt:
       # However, we need to take into account that there may be some
-      # noise in the data, so take 10th percentile and multiply by 2.
+      # noise in the data, so threshold needs to be above zero
       x = activity
-      activityThreshold = quantile(x = x, probs = 0.1) * 2
+
+      # smooth x to 5 minute rolling average to reduce sensitivity to sudden peaks
+      ma <- function(x, n = 300 / ws3){stats::filter(x, rep(1 / n, n), sides = 2, circular = TRUE)}
+      x = ma(x)
+      activityThreshold = sd(x, na.rm = TRUE) * 0.2
       if (HASPT.ignore.invalid == TRUE) {
         invalid = adjustlength(x, invalid)
         zeroMovement = which(x <= activityThreshold & invalid == 0)

R/g.part5.R

@@ -327,10 +327,12 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
               }
               shortendFname = gsub(pattern = "[.]|RData|csv|cwa|bin", replacement = "", x = fnames.ms3[i], ignore.case = TRUE)
               sibreport_fname =  paste0(metadatadir,ms5.sibreport,"/sib_report_", shortendFname, "_",sibDef,".csv")
-              data.table::fwrite(x = sibreport, file = sibreport_fname, row.names = FALSE,
-                                 sep = params_output[["sep_reports"]])
+              if (length(sibreport) > 0) {
+                data.table::fwrite(x = sibreport, file = sibreport_fname, row.names = FALSE,
+                                   sep = params_output[["sep_reports"]])
+              }
               # nap/sib/nonwear overlap analysis
-              if (length(params_sleep[["nap_model"]]) > 0) {
+              if (length(params_sleep[["nap_model"]]) > 0 & length(sibreport) > 0) {
                 # nap detection
                 if (params_general[["acc.metric"]] != "ENMO" |
                     params_sleep[["HASIB.algo"]] != "vanHees2015") {

R/g.plot5.R

@@ -136,13 +136,15 @@ g.plot5 = function(metadatadir = c(), dofirstpage = FALSE, viewingwindow = 1,
             n2exclude = sort(unique(c(n2exclude, missingNights)))
           }
           # Account for shift in nights relative to windows
-          if (P2daysummary_tmp$`N hours`[1] < 24 & P2daysummary_tmp$`N hours`[1] > 12 & length(n2exclude) > 0) {
-            # First calendar day is between 12 and 24 hours
-            # this means that the first night viewindow (=2) may include some data but
-            # is not reflected by the sleep reportswindow
-            # so our night counts should be increased by one
-            n2excludeb = n2exclude = n2exclude + 1
-            incrementNight = 1
+          if (nrow(P2daysummary_tmp) > 0) {
+            if (P2daysummary_tmp$`N hours`[1] < 24 & P2daysummary_tmp$`N hours`[1] > 12 & length(n2exclude) > 0) {
+              # First calendar day is between 12 and 24 hours
+              # this means that the first night viewindow (=2) may include some data but
+              # is not reflected by the sleep reportswindow
+              # so our night counts should be increased by one
+              n2excludeb = n2exclude = n2exclude + 1
+              incrementNight = 1
+            }
           }
 
           if (length(tail_expansion_log) != 0) { # then keep timing of sleeponset to plot

R/g.report.part5.R

@@ -118,6 +118,18 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
         colnames(tmp) = expectedCols
         out = base::merge(tmp, out, all = TRUE)
       }
+      if (!is.null(expectedCols) && ncol(out) > length(expectedCols)) {
+        warning(paste0("Columns dropped in output part5 for ",
+                       basename(x),
+                       " because these could not be matched to columns for earlier",
+                       " recordings in this dataset. Please check whether these recordings ",
+                       " were processed with a",
+                       " different GGIR version or configuration. If yes, reprocess",
+                       " consistently. If not, ",
+                       " consider renaming this file such that it is",
+                       " alphabetically first and by that processed first, which",
+                       " should address the issue."), call. = FALSE)
+      }
       if (length(tail_expansion_log) != 0) {
         col2na = grep(pattern = paste0("sleep_efficiency|N_atleast5minwakenight|daysleeper|",
                                        "daysleeper|sleeplog_used|_spt_sleep|_spt_wake"),
@@ -130,8 +142,26 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
       }
       return(out)
     }
-    out_try = myfun(fnames.ms5[f0])
-    expectedCols = colnames(out_try)
+    # It can be that first part 5 milestone file has less columns when it 
+    # was processed with a different GGIR version or configuration.
+    # This would then complicate
+    # appending later milestone files to it. So, check first a random sample for highest number
+    # of columns. If this fails to identify a file with the maximum number of
+    # column then the above function will provide a warning to
+    # inform the user and with instructions on how to address it.
+    expectedCols = NULL
+    set.seed(1234)
+    # Create list of file indices to try
+    testfiles = unique(sample(x = f0:f1, size = pmin(5, f1 - f0 + 1)))
+    for (testf in testfiles) {
+      out_try = myfun(fnames.ms5[testf])
+      if (ncol(out_try) > length(expectedCols)) {
+        # expectedCols should equal the column names of the milestone
+        # file with the largest number of columns
+        expectedCols = colnames(out_try) 
+      }
+    }
+    # Now load and append all files
     outputfinal = as.data.frame(do.call(rbind,
                                         lapply(fnames.ms5[f0:f1], myfun, expectedCols)),
                                 stringsAsFactors = FALSE)

README.md

@@ -1,7 +1,7 @@
 ![](vignettes/GGIR-MASTERLOGO-RGB.png)
 
 ![GitHub Actions R-CMD-check](https://github.com/wadpac/GGIR/workflows/R-CMD-check/badge.svg)
-[![codecov](https://codecov.io/gh/wadpac/GGIR/branch/master/graph/badge.svg)](https://app.codecov.io/gh/wadpac/GGIR)
+[![codecov](https://codecov.io/gh/wadpac/GGIR/graph/badge.svg?token=p5Po3OpEPG)](https://codecov.io/gh/wadpac/GGIR)
 [![](https://cranlogs.r-pkg.org/badges/last-month/GGIR)](https://cran.r-project.org/package=GGIR)
 
 ## Getting started:

man/GGIR.Rd

@@ -1105,7 +1105,9 @@ GGIR(mode = 1:5,
         "HorAngle", which is based on HDCZA but replaces non-movement detection of
         the HDCZA algorithm by looking for time segments where the angle of the
         longitudinal sensor axis has an angle relative to the horizontal plane
-        between -45 and +45 degrees.}
+        between -45 and +45 degrees. And "NotWorn" which is also the same as HDCZA
+        but looks for time segments when the 5 minute rolling average of counts is
+        below 20 per cent of its standard deviation.}
 
       \item{HASPT.ignore.invalid}{
         Boolean (default = FALSE).

vignettes/CutPoints.Rmd

@@ -274,7 +274,27 @@ activity intensity classification with cut-points.
 |                  | Hip                | (mean: 78.7 yr)   | **`do.enmo = TRUE`\    | Moderate: 14\   |
 |                  |                    |                   | `acc.metric = "ENMO"`  | Vigorous: N/A   |
 +------------------+--------------------+-------------------+------------------------+-----------------+
-| Fraysse 2020^†^  | GENEActiv\        | ≥70 yr \          | `do.enmoa = TRUE`\     | Light: 42.5\    |
+| Bammann 2021     | ActiGraph\         | 62.9 (SD=3.6) yr  | **Default values\      | Moderate: 94\   |
+|                  | Hip                |                   | **`do.enmo = TRUE`\    | Vigorous: 230   |
+|                  |                    |                   | `acc.metric = "ENMO"`  |                 |
++------------------+--------------------+-------------------+------------------------+-----------------+
+| Bammann 2021     | ActiGraph\         | 62.9 (SD=3.6) yr  | **Default values\      | Moderate: 122\  |
+|                  | Dominant Wrist     |                   | **`do.enmo = TRUE`\    | Vigorous: 234   |
+|                  |                    |                   | `acc.metric = "ENMO"`  |                 |
++------------------+--------------------+-------------------+------------------------+-----------------+
+| Bammann 2021     | ActiGraph\         | 62.9 (SD=3.6) yr  | **Default values\      | Moderate: 100\  |
+|                  | Non-dominant Wrist |                   | **`do.enmo = TRUE`\    | Vigorous: 245   |
+|                  |                    |                   | `acc.metric = "ENMO"`  |                 |
++------------------+--------------------+-------------------+------------------------+-----------------+
+| Bammann 2021     | ActiGraph\         | 62.9 (SD=3.6) yr  | **Default values\      | Moderate: 342   |
+|                  | Dominant Ankle     |                   | **`do.enmo = TRUE`\    |                 |
+|                  |                    |                   | `acc.metric = "ENMO"`  |                 |
++------------------+--------------------+-------------------+------------------------+-----------------+
+| Bammann 2021     | ActiGraph\         | 62.9 (SD=3.6) yr  | **Default values\      | Moderate: 331   |
+|                  | Non-dominant Ankle |                   | **`do.enmo = TRUE`\    |                 |
+|                  |                    |                   | `acc.metric = "ENMO"`  |                 |
++------------------+--------------------+-------------------+------------------------+-----------------+
+| Fraysse 2020^†^  | GENEActiv\         | ≥70 yr \          | `do.enmoa = TRUE`\     | Light: 42.5\    |
 |                  | Non-dominant wrist | (mean: 77 yr)     | `do.enmo = FALSE`\     | Moderate: 98\   |
 |                  |                    |                   | `acc.metric = "ENMOa"` | Vigorous: N/A   |
 +------------------+--------------------+-------------------+------------------------+-----------------+
@@ -307,6 +327,7 @@ activity intensity classification with cut-points.
 |                  |                    |                   | `acc.metric = "MAD"`   | Vigorous: N/A   |
 +------------------+--------------------+-------------------+------------------------+-----------------+
 
+
 \*Cut-points derived from applying the Youden index on ROC curves.\
 \*\* Cut-points derived from increasing Sensitivity over Specificity for light and vice versa for moderate on ROC curves (see [paper](https://doi.org/10.1080/02640414.2018.1555904) for more details).\
 ^†^ These publications used acceleration metrics that sum their values per epoch rather than average them per epoch like GGIR does. So, to use their cut-point in GGIR, we provide a scaled version of the cut-points presented in the paper as: `(CutPointFromPaper_in_gmins/(sampleRateFromPaper * EpochLengthInSecondsPaper)) * 1000`
@@ -341,16 +362,22 @@ why cut-points still have value in GGIR see:
 
 # References
 
+-   Aittasalo 2015: <https://bmcsportsscimedrehabil.biomedcentral.com/articles/10.1186/s13102-015-0010-0>
+-   Bammann 2021: <https://doi.org/10.1371/journal.pone.0252615>
+-   Dibben 2020: <https://bmcsportsscimedrehabil.biomedcentral.com/articles/10.1186/s13102-020-00196-7>
+-   Dillon 2016: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4858250/>
 -   Esliger 2011: <https://journals.lww.com/acsm-msse/Fulltext/2011/06000/Validation_of_the_GENEA_Accelerometer.22.aspx>
--   Phillips 2013: <https://doi.org/10.1016/j.jsams.2012.05.013>
--   Schaefer 2014: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3960318/>
+-   Fraysse 2020: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7843957/>
 -   Hildebrand 2014: <https://journals.lww.com/acsm-msse/Fulltext/2014/09000/Age_Group_Comparability_of_Raw_Accelerometer.17.aspx>
 -   Hildebrand 2016: <https://doi.org/10.1111/sms.12795>
--   Vähä-Ypyä 2015: <https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0134813>
--   Aittasalo 2015: <https://bmcsportsscimedrehabil.biomedcentral.com/articles/10.1186/s13102-015-0010-0>
--   Roscoe 2017: <https://link.springer.com/article/10.1007/s00431-017-2948-2>
--   Sanders 2018: <https://doi.org/10.1080/02640414.2018.1555904>
 -   Migueles 2021: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8150960/>
--   Fraysse 2020: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7843957/>
--   Dibben 2020: <https://bmcsportsscimedrehabil.biomedcentral.com/articles/10.1186/s13102-020-00196-7>
--   Dillon 2016: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4858250/>
+-   Phillips 2013: <https://doi.org/10.1016/j.jsams.2012.05.013>
+-   Sanders 2018: <https://doi.org/10.1080/02640414.2018.1555904>
+-   Schaefer 2014: <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3960318/>
+-   Roscoe 2017: <https://link.springer.com/article/10.1007/s00431-017-2948-2>
+-   Vähä-Ypyä 2015: <https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0134813>
+
+
+
+
+