From bb5579fd6dd9c89c923d6f7219ea0cddde7d0d81 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 8 May 2023 16:44:05 +0200 Subject: [PATCH 01/11] #14 skip data reading if gap to last timestamp is more than 5 seconds --- R/readAxivity.R | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index 00f6799..2e1f593 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -46,7 +46,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire ############################################################################# # Internal functions - timestampDecoder = function(coded, fraction, shift, struc) { + timestampDecoder = function(coded, fraction, shift, struc, configtz) { year = struc[[1]] if (year == 0) { # Extract parts of date @@ -70,7 +70,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire struc <- list(year,secs) # Add fractional part and shift start = year + fraction / 65536 + shift - invisible(list(start = start, struc = struc)) } @@ -102,7 +101,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire # data is matrix with three columns "x", "y", and "z" # matrix data is presented if complete == TRUE only. # - if (!is.null(parameters)) { accelScaleCode = parameters$accelScaleCode accelScale = parameters$accelScale @@ -241,8 +239,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire frequency_data = frequency_data, format = format) } - - tsDeco = timestampDecoder(timeStamp, fractional, -shift / frequency_data, struc) + tsDeco = timestampDecoder(timeStamp, fractional, -shift / frequency_data, struc, configtz) start = tsDeco$start struc = tsDeco$struc rawdata_list = list( @@ -408,9 +405,16 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire rawAccel = matrix(0, nrow = 300 * Npages, ncol = 6) } rawPos = 1 + deltaStartLast = 0 for (i in 2:numDBlocks) { + struc_before = struc + if (deltaStartLast < -5) { + complete = FALSE # Do not attempt to read data if gap relative to last timestamp is still more than 5 seconds + } else { + complete = TRUE + } raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, - parameters = prevRaw$parameters) + parameters = prevRaw$parameters, complete = TRUE) if (is.null(raw)) { break } @@ -418,6 +422,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire prevStart = prevRaw$start prevLength = prevRaw$length # Check are previous data block necessary + deltaStartLast = raw$start - start if (raw$start < start) { prevRaw = raw next From a7d71aea94524f06cee46c0265d637f0695840ee Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 8 May 2023 19:22:32 +0200 Subject: [PATCH 02/11] #14 small edits to optimise code --- R/readAxivity.R | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index 2e1f593..a4bb7e3 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -112,7 +112,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire # Check the block header # idstr = readChar(fid,2,useBytes = TRUE) # seek(fid, 2, origin = 'current') # idstr and assume it is AX - # Read the data block. Extract several data fields # offset 4 contains u16 with timestamp offset seek(fid, 4, origin = 'current') # skip packetlength @@ -122,8 +121,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire timeStamp = readBin(fid, integer(), size = 4) # Get light u16 in offset 18 offset18 = unsigned16(readBin(fid, integer(), size = 2)) - # light = 2 ^ (3.0 * (offset18 / 512.0 + 1.0)) # used for AX3, but this seems to have been incorrect - light = bitwAnd(offset18, 0x03ffL) # this seems to match better what is shown in OMGUI + light = bitwAnd(offset18, 0x03ffL) # Read and recalculate temperature u16 in offset 20 temperature = (150.0 * readBin(fid, integer(), size = 2) - 20500.0) / 1000.0; if (loadbattery == TRUE) { @@ -165,13 +163,14 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire if (is.null(parameters)) { if (bitwAnd(tsOffset, 0x8000L) != 0) { format = 1 - } else if (bitwAnd(tsOffset, 0x8000L) == 0) { # & class(frequency_data) == "function") { + frequency_data = round( 3200 / bitwShiftL(1, 15 - bitwAnd(samplerate_dynrange, 15))) + accrange = bitwShiftR(16,(bitwShiftR(abs(samplerate_dynrange),6))) + } else { # & class(frequency_data) == "function") { format = 2 + frequency_data = round( 3200 / bitwShiftL(1, 15 - bitwAnd(samplerate_dynrange, 15))) } } if (format == 1) { - frequency_data = round( 3200 / bitwShiftL(1, 15 - bitwAnd(samplerate_dynrange, 15))) - accrange = bitwShiftR(16,(bitwShiftR(abs(samplerate_dynrange),6))) # Need to undo backwards-compatible shim: # Take into account how many whole samples the fractional part # of timestamp accounts for: @@ -183,12 +182,9 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire # [0][1][2][3][4][5][6][7][8][9] # use 15-bits as 16-bit fractional time fractional = bitwShiftL(bitwAnd(tsOffset, 0x7fffL), 1); + # frequency is truncated to int in firmware shift = shift + bitwShiftR((fractional * frequency_data), 16); - } else if (format == 2) { # & class(frequency_data) == "function") { - if (is.null(parameters)) { - frequency_data = round( 3200 / bitwShiftL(1, 15 - bitwAnd(samplerate_dynrange, 15))) - } } } else { #Very old format, where offset 26 contains frequency @@ -196,6 +192,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire } # Read data if necessary if (complete) { + if (packed) { #32 bit # Read 4 byte for three measurements packedData = readBin(fid, integer(), size = 4, n = blockLength) @@ -207,7 +204,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire } else { # Read unpacked data xyz = readBin(fid, integer(), size = 2, n = blockLength * Naxes) #*3 - data = matrix(xyz, ncol = Naxes,byrow = T) #3 + data = matrix(xyz, ncol = Naxes, byrow = T, dimnames = FALSE) #3 # Calculate number of bytes to skip temp = 482 - (2 * Naxes * blockLength) } @@ -255,6 +252,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire if (complete) { rawdata_list$data = data } + return(invisible(rawdata_list)) } readHeader = function(fid, numDBlocks) { @@ -405,16 +403,9 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire rawAccel = matrix(0, nrow = 300 * Npages, ncol = 6) } rawPos = 1 - deltaStartLast = 0 for (i in 2:numDBlocks) { - struc_before = struc - if (deltaStartLast < -5) { - complete = FALSE # Do not attempt to read data if gap relative to last timestamp is still more than 5 seconds - } else { - complete = TRUE - } raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, - parameters = prevRaw$parameters, complete = TRUE) + parameters = prevRaw$parameters) if (is.null(raw)) { break } @@ -422,7 +413,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire prevStart = prevRaw$start prevLength = prevRaw$length # Check are previous data block necessary - deltaStartLast = raw$start - start if (raw$start < start) { prevRaw = raw next From dda434d1d0f302ddcf517b1a00896324b9e18504 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 12:00:39 +0200 Subject: [PATCH 03/11] minor bug fix --- R/readAxivity.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index a4bb7e3..4468575 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -412,12 +412,13 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire # Save start and length of the previous block prevStart = prevRaw$start prevLength = prevRaw$length + struc = raw$struc # Check are previous data block necessary if (raw$start < start) { prevRaw = raw next } - struc = raw$struc + # Create array of times time = seq(prevStart, raw$start, length.out = prevLength + 1) From e3a7d8067e460fb8cb1dedd11b6ce56ca6c0aa53 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 12:50:15 +0200 Subject: [PATCH 04/11] code now skips blocks that are not within timerange --- R/readAxivity.R | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index 4468575..c12d06b 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -366,7 +366,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire return(invisible(list(header = header, data = NULL))) } ############################################################################# - # reinitiate file and start reading of data and sesrch the beginning of required + # reinitiate file and start reading of data and search the beginning of required seek(fid,0) # skip header seek(fid, 1024, origin = 'current') @@ -403,9 +403,22 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire rawAccel = matrix(0, nrow = 300 * Npages, ncol = 6) } rawPos = 1 + + # Pseudo code for possible revision: + # - Extract sector/block size in bytes => see c-link https://github.com/digitalinteraction/openmovement/blob/d8678127c8331196072215a2f1a3dfe7fa595915/Software/AX3/cwa-convert/c/main.c#L278 + # - Op basis van block_start do seek() to skip + # - Run code as shown below for (i in 2:numDBlocks) { - raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, - parameters = prevRaw$parameters) + timeSkip = start - prevRaw$start + blockDur = prevRaw$length / prevRaw$frequency + blockSkip = floor(timeSkip/blockDur) - 1 + if (i >= blockSkip) { # start of recording + raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, + parameters = prevRaw$parameters) + } else { + seek(fid, 512, origin = 'current') # skip block + next + } if (is.null(raw)) { break } @@ -418,7 +431,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire prevRaw = raw next } - # Create array of times time = seq(prevStart, raw$start, length.out = prevLength + 1) @@ -467,7 +479,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire break } } - ############################################################################# # Process the last block of data if necessary if (pos <= nr) { # & ignorelastblock == FALSE) { From 03882d8487eb179ea3885b464c8231628ce2c328 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 13:13:55 +0200 Subject: [PATCH 05/11] skip attempt to read last block if start was after end of recording --- R/readAxivity.R | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index c12d06b..7eb2744 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -404,14 +404,11 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire } rawPos = 1 - # Pseudo code for possible revision: - # - Extract sector/block size in bytes => see c-link https://github.com/digitalinteraction/openmovement/blob/d8678127c8331196072215a2f1a3dfe7fa595915/Software/AX3/cwa-convert/c/main.c#L278 - # - Op basis van block_start do seek() to skip - # - Run code as shown below for (i in 2:numDBlocks) { timeSkip = start - prevRaw$start blockDur = prevRaw$length / prevRaw$frequency blockSkip = floor(timeSkip/blockDur) - 1 + if (i >= blockSkip) { # start of recording raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, parameters = prevRaw$parameters) @@ -481,8 +478,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire } ############################################################################# # Process the last block of data if necessary - if (pos <= nr) { # & ignorelastblock == FALSE) { - # print("last block of data") + if (pos <= nr & exists("prevStart") & exists("prevLength")) { # Calculate pseudo time for the "next" block newTimes = (prevRaw$start - prevStart) / prevLength * prevRaw$length + prevRaw$start prevLength = prevRaw$length From 852f98ff6031b69f2f076cf25e766de3b80ff175 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 13:48:49 +0200 Subject: [PATCH 06/11] replace for by while loop such that we skip a whole series of blocks in one go --- R/readAxivity.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index 7eb2744..e187bb4 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -403,8 +403,8 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire rawAccel = matrix(0, nrow = 300 * Npages, ncol = 6) } rawPos = 1 - - for (i in 2:numDBlocks) { + i = 2 + while (i <= numDBlocks) { timeSkip = start - prevRaw$start blockDur = prevRaw$length / prevRaw$frequency blockSkip = floor(timeSkip/blockDur) - 1 @@ -412,8 +412,11 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire if (i >= blockSkip) { # start of recording raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, parameters = prevRaw$parameters) + i = i + 1 } else { - seek(fid, 512, origin = 'current') # skip block + # skip blocks + seek(fid, 512 * blockSkip, origin = 'current') + i = i + blockSkip next } if (is.null(raw)) { @@ -426,6 +429,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire # Check are previous data block necessary if (raw$start < start) { prevRaw = raw + i = i + 1 next } # Create array of times From 16bcf070e3117c7c8856cb10c08d799f6caf9ec8 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 13:54:10 +0200 Subject: [PATCH 07/11] improvement to previous commit --- R/readAxivity.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index e187bb4..308cf8f 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -412,7 +412,6 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire if (i >= blockSkip) { # start of recording raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, parameters = prevRaw$parameters) - i = i + 1 } else { # skip blocks seek(fid, 512 * blockSkip, origin = 'current') @@ -479,6 +478,7 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire if (pos > nr) { break } + i = i + 1 } ############################################################################# # Process the last block of data if necessary From ec170e55ec496a4fc1e6fa8be2848a6453885af4 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 14:18:37 +0200 Subject: [PATCH 08/11] update version number and changelog --- DESCRIPTION | 4 ++-- inst/NEWS.Rd | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 09df9e9..10966f7 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: GGIRread Type: Package Title: Wearable Accelerometer Data File Readers -Version: 0.2.6 -Date: 2022-12-05 +Version: 0.2.7 +Date: 2022-05-12 Authors@R: c(person("Vincent T","van Hees",role=c("aut","cre"), email="v.vanhees@accelting.com"), person(given = "Patrick",family = "Bos", diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd index 348a0ef..e8e51d0 100755 --- a/inst/NEWS.Rd +++ b/inst/NEWS.Rd @@ -1,6 +1,11 @@ \name{NEWS} \title{News for Package \pkg{GGIRread}} \newcommand{\cpkg}{\href{http://CRAN.R-project.org/package=#1}{\pkg{#1}}} +\section{Changes in version 0.2.7 (release date:12-05-2022)}{ + \itemize{ + \item Speeding up readAxivity function by approximately 65 percent. + } +} \section{Changes in version 0.2.6 (release date:05-12-2022)}{ \itemize{ \item New CRAN release, only minor improvements to syntax From fbd272e9f42c5d8b695dbea343a0435c57989240 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 15:01:12 +0200 Subject: [PATCH 09/11] remove dimnames arguments as added value is not clear --- R/readAxivity.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index 308cf8f..a0d818c 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -203,8 +203,8 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire temp = 482 - 4 * (Naxes/3) * blockLength } else { # Read unpacked data - xyz = readBin(fid, integer(), size = 2, n = blockLength * Naxes) #*3 - data = matrix(xyz, ncol = Naxes, byrow = T, dimnames = FALSE) #3 + xyz = readBin(fid, integer(), size = 2, n = blockLength * Naxes) + data = matrix(xyz, ncol = Naxes, byrow = T) # Calculate number of bytes to skip temp = 482 - (2 * Naxes * blockLength) } From 882290d19665d90a3416ae3c000a09dc534a2535 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 15:46:26 +0200 Subject: [PATCH 10/11] updating performance gain in changelog --- inst/NEWS.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd index e8e51d0..3dcdd6b 100755 --- a/inst/NEWS.Rd +++ b/inst/NEWS.Rd @@ -3,7 +3,7 @@ \newcommand{\cpkg}{\href{http://CRAN.R-project.org/package=#1}{\pkg{#1}}} \section{Changes in version 0.2.7 (release date:12-05-2022)}{ \itemize{ - \item Speeding up readAxivity function by approximately 65 percent. + \item Speeding up readAxivity function by approximately 75 percent. } } \section{Changes in version 0.2.6 (release date:05-12-2022)}{ From 4b8f0840ed53676f6e4be4776f51875de5496d7a Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 12 May 2023 15:56:55 +0200 Subject: [PATCH 11/11] minor object name improvements --- R/readAxivity.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/R/readAxivity.R b/R/readAxivity.R index a0d818c..70c26f4 100755 --- a/R/readAxivity.R +++ b/R/readAxivity.R @@ -405,17 +405,17 @@ readAxivity = function(filename, start = 0, end = 0, progressBar = FALSE, desire rawPos = 1 i = 2 while (i <= numDBlocks) { - timeSkip = start - prevRaw$start + time2Skip = start - prevRaw$start blockDur = prevRaw$length / prevRaw$frequency - blockSkip = floor(timeSkip/blockDur) - 1 + Nblocks2Skip = floor(time2Skip/blockDur) - 1 - if (i >= blockSkip) { # start of recording + if (i >= Nblocks2Skip) { # start of recording raw = readDataBlock(fid, header_accrange = header$accrange, struc = struc, parameters = prevRaw$parameters) } else { # skip blocks - seek(fid, 512 * blockSkip, origin = 'current') - i = i + blockSkip + seek(fid, 512 * Nblocks2Skip, origin = 'current') + i = i + Nblocks2Skip next } if (is.null(raw)) {