From 207d712ae4ef53b10946dd9eccbd5aa295eaad9e Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 27 Sep 2024 17:30:25 +0200 Subject: [PATCH] add Fitbit json read functionalty #68 --- DESCRIPTION | 2 +- NEWS.md | 2 +- R/readFitbit.R | 85 ++++++++ README.md | 4 +- .../testfiles/calories-1995-06-23_Fitbit.json | 142 +++++++++++++ inst/testfiles/sleep-1995-06-23_Fitbit.json | 200 ++++++++++++++++++ inst/testfiles/steps-1995-06-23_Fitbit.json | 103 +++++++++ man/readFitbit.Rd | 22 ++ tests/testthat/test_readFitbit.R | 29 +++ 9 files changed, 586 insertions(+), 3 deletions(-) create mode 100644 R/readFitbit.R create mode 100644 inst/testfiles/calories-1995-06-23_Fitbit.json create mode 100644 inst/testfiles/sleep-1995-06-23_Fitbit.json create mode 100644 inst/testfiles/steps-1995-06-23_Fitbit.json create mode 100644 man/readFitbit.Rd create mode 100644 tests/testthat/test_readFitbit.R diff --git a/DESCRIPTION b/DESCRIPTION index 2bfb8d6..d339294 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,7 +22,7 @@ URL: https://github.com/wadpac/GGIRread/ BugReports: https://github.com/wadpac/GGIRread/issues License: Apache License (== 2.0) Suggests: testthat -Imports: matlab, bitops, Rcpp (>= 0.12.10), data.table, readxl +Imports: matlab, bitops, Rcpp (>= 0.12.10), data.table, readxl, jsonlite Depends: stats, utils, R (>= 3.5.0) NeedsCompilation: yes LinkingTo: Rcpp diff --git a/NEWS.md b/NEWS.md index 1d6a235..2719601 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,7 +5,7 @@ - Migrate read function for ActiGraph (csv) and Actiwatch (csv/awd) count data to GGIRread #68. - Add function for reading Actical (csv) count data #68. - Add function to read and merge Philips Health Band file pairs (xlsx) #68. - +- Add function for reading Fitbit (json) with sleep, steps or calories #68. # Changes in version 1.0.1 (release date:03-06-2024) diff --git a/R/readFitbit.R b/R/readFitbit.R new file mode 100644 index 0000000..99f440b --- /dev/null +++ b/R/readFitbit.R @@ -0,0 +1,85 @@ +readFitbit = function(filename = NULL) { + # Assumptions made: + # - sleep is sampled at 30 second resolution + # - steps are sampled at 60 second resolution + # - timestamp format is always the same per data type + + # Declare local functions + handleTimeGaps = function(df, epochSize) { + timeRange = range(df$dateTime) + startTime = timeRange[1] + endTime = timeRange[2] + timeFrame = data.frame(dateTime = seq(startTime, endTime, by = epochSize)) + df = merge(df, timeFrame, by = c("dateTime"), all.x = TRUE) + return(df) + } + + #------------------------------------------------- + # Main code + + D = jsonlite::read_json(path = filename, + simplifyVector = FALSE, + flatten = FALSE) + + # extract dataType as json structure differs between types + dataType = tolower(unlist(strsplit(basename(filename), "-"))[1]) + + if (dataType == "sleep") { + epochSize = 30 + # Put all data in data.frame + for (i in 1:length(D)) { + tmp = D[[i]][15]$levels + data = as.data.frame(data.table::rbindlist(tmp$data, fill = TRUE)) + data$dateTime = as.POSIXct(data$dateTime, format = "%Y-%m-%dT%H:%M:%S") + if (i == 1) { + all_data = data + } else { + all_data = rbind(all_data, data) + } + if ("shortData" %in% names(tmp)) { + shortData = data.table::rbindlist(tmp$shortData, fill = TRUE) + shortData$dateTime = as.POSIXct(shortData$dateTime, format = "%Y-%m-%dT%H:%M:%S") + if (i == 1) { + all_shortData = shortData + } else { + all_shortData = rbind(all_shortData, shortData) + } + } + } + # Expand to full time series + D = as.data.frame(lapply(all_data, rep, all_data$seconds/epochSize)) + D$dateTime = seq(from = D$dateTime[1], length.out = nrow(D), by = epochSize) + D$seconds = epochSize + D = handleTimeGaps(D, epochSize) # Handle time gaps, if any + + S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/30)) + S$dateTime = seq(from = S$dateTime[1], length.out = nrow(S), by = 30) + S$seconds = epochSize + + # merge in shortData (S) + matching_times = which(S$dateTime %in% D$dateTime == TRUE) + non_matching_times = which(S$dateTime %in% D$dateTime == FALSE) + if (length(matching_times) > 0) { + times_to_replace = S$dateTime[matching_times] + D[which(D$dateTime %in% times_to_replace), ] = S[matching_times,] + } + if (length(non_matching_times) > 0) { + D = rbind(D, S[non_matching_times,]) + } + D = handleTimeGaps(D, epochSize) # Handle new time gaps, if any + + # Order time stamps + D = D[order(D$dateTime), ] + colnames(D)[2] = "sleeplevel" + } else if (dataType == "steps" || dataType == "calories") { + epochSize = 60 + data = as.data.frame(data.table::rbindlist(D, fill = TRUE)) + data$dateTime = as.POSIXct(data$dateTime, format = "%m/%d/%y %H:%M:%S") + D = handleTimeGaps(data, epochSize = 60) + D$value = as.numeric(D$value) + colnames(D)[2] = dataType + } else { + stop("File type not recognised") + } + return(D) +} \ No newline at end of file diff --git a/README.md b/README.md index 915fd8c..8cc1940 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,6 @@ ActivInsights Ltd https://activinsights.com/ | GENEActiv Original and Sleep | .b Unilever Discover Ltd | Genea (no longer manufactured) | .bin | raw gravitational units | readGenea ActiGraph | ??? | .csv | count data | readActigraph Actiwatch | ??? | .csv and .awd | count data | readActiwatch -Actical | ??? | .csv | count data | readActical \ No newline at end of file +Actical | ??? | .csv | count data | readActical.R +Philips Health Band | ??? | .xlsx | count data | mergePHBfilePairs.R +Fitbit | ??? | .json | sleep, steps or calories data | readFitbit.R diff --git a/inst/testfiles/calories-1995-06-23_Fitbit.json b/inst/testfiles/calories-1995-06-23_Fitbit.json new file mode 100644 index 0000000..81f1294 --- /dev/null +++ b/inst/testfiles/calories-1995-06-23_Fitbit.json @@ -0,0 +1,142 @@ +[{ + "dateTime" : "06/23/95 00:00:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:01:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:02:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:03:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:04:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:05:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:06:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:07:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:08:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:09:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:10:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:11:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:12:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:13:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:14:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:15:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:16:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:17:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:18:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:19:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:20:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:21:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:22:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:23:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:24:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:25:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:26:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:27:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:28:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:29:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:30:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:31:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:32:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:33:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:34:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:35:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:36:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:37:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:38:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:39:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:40:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:41:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:42:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:43:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:44:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:45:00", + "value" : "1.48" +},{ + "dateTime" : "06/23/95 00:46:00", + "value" : "1.48" +}] \ No newline at end of file diff --git a/inst/testfiles/sleep-1995-06-23_Fitbit.json b/inst/testfiles/sleep-1995-06-23_Fitbit.json new file mode 100644 index 0000000..e9ba306 --- /dev/null +++ b/inst/testfiles/sleep-1995-06-23_Fitbit.json @@ -0,0 +1,200 @@ +[{ + "logId" : 12345, + "dateOfSleep" : "1995-07-11", + "startTime" : "1995-07-11T02:28:30.000", + "endTime" : "1995-07-11T06:57:00.000", + "duration" : 16080000, + "minutesToFallAsleep" : 0, + "minutesAsleep" : 236, + "minutesAwake" : 32, + "minutesAfterWakeup" : 0, + "timeInBed" : 268, + "efficiency" : 96, + "type" : "stages", + "infoCode" : 0, + "logType" : "auto_detected", + "levels" : { + "summary" : { + "deep" : { + "count" : 4, + "minutes" : 59, + "thirtyDayAvgMinutes" : 45 + }, + "wake" : { + "count" : 11, + "minutes" : 32, + "thirtyDayAvgMinutes" : 45 + }, + "light" : { + "count" : 14, + "minutes" : 143, + "thirtyDayAvgMinutes" : 213 + }, + "rem" : { + "count" : 4, + "minutes" : 34, + "thirtyDayAvgMinutes" : 64 + } + }, + "data" : [{ + "dateTime" : "1995-07-11T02:28:30.000", + "level" : "light", + "seconds" : 2160 + },{ + "dateTime" : "1995-07-11T03:04:30.000", + "level" : "deep", + "seconds" : 300 + },{ + "dateTime" : "1995-07-11T03:09:30.000", + "level" : "light", + "seconds" : 450 + },{ + "dateTime" : "1995-07-11T03:17:00.000", + "level" : "deep", + "seconds" : 1080 + },{ + "dateTime" : "1995-07-11T03:35:00.000", + "level" : "light", + "seconds" : 240 + },{ + "dateTime" : "1995-07-11T03:39:00.000", + "level" : "deep", + "seconds" : 1830 + },{ + "dateTime" : "1995-07-11T04:09:30.000", + "level" : "light", + "seconds" : 2670 + },{ + "dateTime" : "1995-07-11T04:54:00.000", + "level" : "deep", + "seconds" : 390 + },{ + "dateTime" : "1995-07-11T05:00:30.000", + "level" : "light", + "seconds" : 750 + },{ + "dateTime" : "1995-07-11T05:13:00.000", + "level" : "rem", + "seconds" : 270 + },{ + "dateTime" : "1995-07-11T05:17:30.000", + "level" : "light", + "seconds" : 1050 + },{ + "dateTime" : "1995-07-11T05:35:00.000", + "level" : "wake", + "seconds" : 600 + },{ + "dateTime" : "1995-07-11T05:45:00.000", + "level" : "light", + "seconds" : 240 + },{ + "dateTime" : "1995-07-11T05:49:00.000", + "level" : "wake", + "seconds" : 660 + },{ + "dateTime" : "1995-07-11T06:00:00.000", + "level" : "light", + "seconds" : 930 + },{ + "dateTime" : "1995-07-11T06:15:30.000", + "level" : "rem", + "seconds" : 300 + },{ + "dateTime" : "1995-07-11T06:20:30.000", + "level" : "light", + "seconds" : 630 + },{ + "dateTime" : "1995-07-11T06:31:00.000", + "level" : "rem", + "seconds" : 1560 + }], + "shortData" : [{ + "dateTime" : "1995-07-11T04:08:30.000", + "level" : "wake", + "seconds" : 60 + },{ + "dateTime" : "1995-07-11T04:15:30.000", + "level" : "wake", + "seconds" : 150 + },{ + "dateTime" : "1995-07-11T04:33:30.000", + "level" : "wake", + "seconds" : 60 + },{ + "dateTime" : "1995-07-11T05:00:30.000", + "level" : "wake", + "seconds" : 60 + },{ + "dateTime" : "1995-07-11T06:13:30.000", + "level" : "wake", + "seconds" : 90 + },{ + "dateTime" : "1995-07-11T06:22:30.000", + "level" : "wake", + "seconds" : 90 + },{ + "dateTime" : "1995-07-11T06:26:00.000", + "level" : "wake", + "seconds" : 120 + },{ + "dateTime" : "1995-07-11T06:33:30.000", + "level" : "wake", + "seconds" : 30 + },{ + "dateTime" : "1995-07-11T06:56:30.000", + "level" : "wake", + "seconds" : 30 + }] + }, + "mainSleep" : true +},{ + "logId" : 12345, + "dateOfSleep" : "1995-06-25", + "startTime" : "1995-06-24T22:47:30.000", + "endTime" : "1995-06-25T00:06:30.000", + "duration" : 4740000, + "minutesToFallAsleep" : 0, + "minutesAsleep" : 73, + "minutesAwake" : 6, + "minutesAfterWakeup" : 0, + "timeInBed" : 79, + "efficiency" : 92, + "type" : "classic", + "infoCode" : 2, + "logType" : "auto_detected", + "levels" : { + "summary" : { + "restless" : { + "count" : 1, + "minutes" : 5 + }, + "awake" : { + "count" : 1, + "minutes" : 1 + }, + "asleep" : { + "count" : 0, + "minutes" : 73 + } + }, + "data" : [{ + "dateTime" : "1995-06-24T22:47:30.000", + "level" : "asleep", + "seconds" : 60 + },{ + "dateTime" : "1995-06-24T22:48:30.000", + "level" : "restless", + "seconds" : 300 + },{ + "dateTime" : "1995-06-24T22:53:30.000", + "level" : "awake", + "seconds" : 60 + },{ + "dateTime" : "1995-06-24T22:54:30.000", + "level" : "asleep", + "seconds" : 4320 + }] + }, + "mainSleep" : false +}] \ No newline at end of file diff --git a/inst/testfiles/steps-1995-06-23_Fitbit.json b/inst/testfiles/steps-1995-06-23_Fitbit.json new file mode 100644 index 0000000..e9785f8 --- /dev/null +++ b/inst/testfiles/steps-1995-06-23_Fitbit.json @@ -0,0 +1,103 @@ +[{ + "dateTime" : "06/24/95 16:00:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:01:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:02:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:03:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:04:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:06:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:08:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:09:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:10:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:11:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:12:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:13:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:14:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:15:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:16:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:17:00", + "value" : "7" +},{ + "dateTime" : "06/24/95 16:18:00", + "value" : "44" +},{ + "dateTime" : "06/24/95 16:19:00", + "value" : "12" +},{ + "dateTime" : "06/24/95 16:20:00", + "value" : "58" +},{ + "dateTime" : "06/24/95 16:21:00", + "value" : "79" +},{ + "dateTime" : "06/24/95 16:22:00", + "value" : "95" +},{ + "dateTime" : "06/24/95 16:23:00", + "value" : "98" +},{ + "dateTime" : "06/24/95 16:24:00", + "value" : "102" +},{ + "dateTime" : "06/24/95 16:25:00", + "value" : "90" +},{ + "dateTime" : "06/24/95 16:26:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:27:00", + "value" : "22" +},{ + "dateTime" : "06/24/95 16:28:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:29:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:30:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:31:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:32:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:33:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:34:00", + "value" : "0" +},{ + "dateTime" : "06/24/95 16:35:00", + "value" : "0" +}] \ No newline at end of file diff --git a/man/readFitbit.Rd b/man/readFitbit.Rd new file mode 100644 index 0000000..8d5a323 --- /dev/null +++ b/man/readFitbit.Rd @@ -0,0 +1,22 @@ +\name{readFitbit} +\alias{readFitbit} +\title{ + Read Fitbit data files (json) +} +\description{ + Reads Fitbit data file (json) with sleep, steps or calories. +} +\usage{ + readFitbit(filename = NULL) +} +\arguments{ + \item{filename}{ + Character, filename (required) of json file + } +} +\value{ + Data.frame with data converted to time series +} +\author{ + Vincent T van Hees +} \ No newline at end of file diff --git a/tests/testthat/test_readFitbit.R b/tests/testthat/test_readFitbit.R new file mode 100644 index 0000000..eff3235 --- /dev/null +++ b/tests/testthat/test_readFitbit.R @@ -0,0 +1,29 @@ +library(GGIRread) +context("read Fitbit json") +test_that("Fitbit json is correctly read", { + # Sleep + file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") + D = readFitbit(filename = file) + expect_equal(nrow(D), 695) + expect_equal(ncol(D), 3) + expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30") + TB = table(D$sleeplevel) + expect_equal(names(TB), c("asleep", "awake", "deep", "light", "rem", "restless", "wake")) + expect_equal(as.numeric(TB), c(146, 2, 118, 283, 71, 10, 65)) + + # Steps + file = system.file("testfiles/steps-1995-06-23_Fitbit.json", package = "GGIRread") + D = readFitbit(filename = file) + expect_equal(nrow(D), 34) + expect_equal(ncol(D), 2) + expect_equal(format(D$dateTime[1]), "1995-06-24 16:00:00") + expect_equal(sum(D$steps), 607) + + # Calories + file = system.file("testfiles/calories-1995-06-23_Fitbit.json", package = "GGIRread") + D = readFitbit(filename = file) + expect_equal(nrow(D), 47) + expect_equal(ncol(D), 2) + expect_equal(format(D$dateTime[1]), "1995-06-23") + expect_equal(sum(D$calories), 69.56) +}) \ No newline at end of file