From 0443d1589212b69eab140369b02eb5ddfcf4fe31 Mon Sep 17 00:00:00 2001 From: mbegalevibrent Date: Wed, 28 Feb 2018 22:29:07 -0600 Subject: [PATCH 01/17] updated repository to include fitbit schemas activity steps, heart rate and sleeep from fitbit API --- .../passive/fitbit/fitbit_activity_steps.avsc | 29 ++++++++++++++++++ commons/passive/fitbit/fitbit_heart_rate.avsc | 30 +++++++++++++++++++ .../passive/fitbit/fitbit_sleep_classic.avsc | 25 ++++++++++++++++ .../passive/fitbit/fitbit_sleep_stages.avsc | 25 ++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 commons/passive/fitbit/fitbit_activity_steps.avsc create mode 100644 commons/passive/fitbit/fitbit_heart_rate.avsc create mode 100644 commons/passive/fitbit/fitbit_sleep_classic.avsc create mode 100644 commons/passive/fitbit/fitbit_sleep_stages.avsc diff --git a/commons/passive/fitbit/fitbit_activity_steps.avsc b/commons/passive/fitbit/fitbit_activity_steps.avsc new file mode 100644 index 00000000..079a6095 --- /dev/null +++ b/commons/passive/fitbit/fitbit_activity_steps.avsc @@ -0,0 +1,29 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitIntradayHeartRate", + "doc": "Intra day heart rate data from fitbit device", + "fields": [ + { + "name": "type", + "type": "string", + "doc": "Type of data coming from fitbit intraday API, (e.g. heart rate)" + },{ + "name": "datasetInterval", + "type": "int", + "doc": "Chronological window size (e.g. 1)" + },{ + "name": "datasetType", + "type": "string", + "doc": "Chronological window unit (e.g. 'minute')" + },{ + "name": "time", + "type": "string", + "doc": "Hours, minutes, seconds of start of interval (e.g. 00:33:00)" + },{ + "name": "value", + "type": "int", + "doc":"Steps taken in this meriod" + } + ] +} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_heart_rate.avsc b/commons/passive/fitbit/fitbit_heart_rate.avsc new file mode 100644 index 00000000..376112e4 --- /dev/null +++ b/commons/passive/fitbit/fitbit_heart_rate.avsc @@ -0,0 +1,30 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitIntradayHeartRate", + "doc": "Intra day heart rate data from fitbit device", + "fields": [ + { + "name": "type", + "type": "string", + "doc": "Type of data coming from fitbit intraday API, (e.g. heart rate)" + }, + { + "name": "datasetInterval", + "type": "int", + "doc": "Chronological window size (e.g. 1)" + },{ + "name": "datasetType", + "type": "string", + "doc": "Chronological window unit (e.g. 'minute')" + },{ + "name": "time", + "type": "string", + "doc": "Hours, minutes, seconds of start of interval (e.g. 00:33:00)" + },{ + "name": "value", + "type": "int", + "doc":"Heart rate BPM value" + } + ] +} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_sleep_classic.avsc b/commons/passive/fitbit/fitbit_sleep_classic.avsc new file mode 100644 index 00000000..9a0d3baa --- /dev/null +++ b/commons/passive/fitbit/fitbit_sleep_classic.avsc @@ -0,0 +1,25 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitSleepClassic", + "doc": "Classic sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/", + "fields": [ + { + "name": "type", + "type": "string", + "doc": "Type of data coming from fitbit intraday API, (e.g. sleep classic)" + },{ + "name": "level", + "type": "string", + "doc": "Level of sleep, 'awake', 'restless', or 'asleep'" + },{ + "name": "dateTime", + "type": "string", + "doc": "Hours, minutes, seconds of start of interval" + },{ + "name": "seconds", + "type": "int", + "doc":"Duration at this sleep characteristic in secondse" + } + ] +} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_sleep_stages.avsc b/commons/passive/fitbit/fitbit_sleep_stages.avsc new file mode 100644 index 00000000..7358f05c --- /dev/null +++ b/commons/passive/fitbit/fitbit_sleep_stages.avsc @@ -0,0 +1,25 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitSleepStages", + "doc": "Fitbit 'stages' sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/", + "fields": [ + { + "name": "type", + "type": "string", + "doc": "Type of data coming from fitbit intraday API, (e.g. sleep classic)" + },{ + "name": "level", + "type": "string", + "doc": "Level of sleep, 'deep', 'light', 'rem', or 'awake'" + },{ + "name": "dateTime", + "type": "string", + "doc": "Hours, minutes, seconds of start of interval" + },{ + "name": "seconds", + "type": "int", + "doc":"Duration at this sleep characteristic in secondse" + } + ] +} \ No newline at end of file From 0ba01ce9471135d708370583bd1359ec0b68d811 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 1 Mar 2018 11:22:11 +0000 Subject: [PATCH 02/17] Snapshot version --- java-sdk/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/build.gradle b/java-sdk/build.gradle index a837dc4a..0823b306 100644 --- a/java-sdk/build.gradle +++ b/java-sdk/build.gradle @@ -17,7 +17,7 @@ subprojects { apply plugin: 'idea' // Configuration - version = '0.3.2' + version = '0.3.3-SNAPSHOT' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Schemas' From f9230cb6875e79524769358250e97e7caa2f4e5a Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 1 Mar 2018 14:59:07 +0000 Subject: [PATCH 03/17] Refactor schemas --- .../passive/fitbit/fitbit_activity_steps.avsc | 29 ------------------ commons/passive/fitbit/fitbit_heart_rate.avsc | 30 ------------------- .../fitbit/fitbit_intraday_heart_rate.avsc | 13 ++++++++ .../passive/fitbit/fitbit_intraday_steps.avsc | 13 ++++++++ .../passive/fitbit/fitbit_sleep_classic.avsc | 25 ---------------- .../passive/fitbit/fitbit_sleep_pattern.avsc | 13 ++++++++ .../passive/fitbit/fitbit_sleep_stage.avsc | 13 ++++++++ .../passive/fitbit/fitbit_sleep_stages.avsc | 25 ---------------- 8 files changed, 52 insertions(+), 109 deletions(-) delete mode 100644 commons/passive/fitbit/fitbit_activity_steps.avsc delete mode 100644 commons/passive/fitbit/fitbit_heart_rate.avsc create mode 100644 commons/passive/fitbit/fitbit_intraday_heart_rate.avsc create mode 100644 commons/passive/fitbit/fitbit_intraday_steps.avsc delete mode 100644 commons/passive/fitbit/fitbit_sleep_classic.avsc create mode 100644 commons/passive/fitbit/fitbit_sleep_pattern.avsc create mode 100644 commons/passive/fitbit/fitbit_sleep_stage.avsc delete mode 100644 commons/passive/fitbit/fitbit_sleep_stages.avsc diff --git a/commons/passive/fitbit/fitbit_activity_steps.avsc b/commons/passive/fitbit/fitbit_activity_steps.avsc deleted file mode 100644 index 079a6095..00000000 --- a/commons/passive/fitbit/fitbit_activity_steps.avsc +++ /dev/null @@ -1,29 +0,0 @@ -{ - "namespace": "org.radarcns.passive.fitbit", - "type": "record", - "name": "FitbitIntradayHeartRate", - "doc": "Intra day heart rate data from fitbit device", - "fields": [ - { - "name": "type", - "type": "string", - "doc": "Type of data coming from fitbit intraday API, (e.g. heart rate)" - },{ - "name": "datasetInterval", - "type": "int", - "doc": "Chronological window size (e.g. 1)" - },{ - "name": "datasetType", - "type": "string", - "doc": "Chronological window unit (e.g. 'minute')" - },{ - "name": "time", - "type": "string", - "doc": "Hours, minutes, seconds of start of interval (e.g. 00:33:00)" - },{ - "name": "value", - "type": "int", - "doc":"Steps taken in this meriod" - } - ] -} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_heart_rate.avsc b/commons/passive/fitbit/fitbit_heart_rate.avsc deleted file mode 100644 index 376112e4..00000000 --- a/commons/passive/fitbit/fitbit_heart_rate.avsc +++ /dev/null @@ -1,30 +0,0 @@ -{ - "namespace": "org.radarcns.passive.fitbit", - "type": "record", - "name": "FitbitIntradayHeartRate", - "doc": "Intra day heart rate data from fitbit device", - "fields": [ - { - "name": "type", - "type": "string", - "doc": "Type of data coming from fitbit intraday API, (e.g. heart rate)" - }, - { - "name": "datasetInterval", - "type": "int", - "doc": "Chronological window size (e.g. 1)" - },{ - "name": "datasetType", - "type": "string", - "doc": "Chronological window unit (e.g. 'minute')" - },{ - "name": "time", - "type": "string", - "doc": "Hours, minutes, seconds of start of interval (e.g. 00:33:00)" - },{ - "name": "value", - "type": "int", - "doc":"Heart rate BPM value" - } - ] -} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_intraday_heart_rate.avsc b/commons/passive/fitbit/fitbit_intraday_heart_rate.avsc new file mode 100644 index 00000000..0ea9876f --- /dev/null +++ b/commons/passive/fitbit/fitbit_intraday_heart_rate.avsc @@ -0,0 +1,13 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitIntradayHeartRate", + "doc": "Intra day heart rate data from fitbit device.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "timeInterval", "type": "int", "doc": "Chronological window size (s)." }, + { "name": "timezoneOffset", "type": "int", "doc": "Offset from UTC (s)." }, + { "name": "heartRate", "type": "int", "doc":"Heart rate value (bpm)."} + ] +} diff --git a/commons/passive/fitbit/fitbit_intraday_steps.avsc b/commons/passive/fitbit/fitbit_intraday_steps.avsc new file mode 100644 index 00000000..4bbbe044 --- /dev/null +++ b/commons/passive/fitbit/fitbit_intraday_steps.avsc @@ -0,0 +1,13 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitIntradaySteps", + "doc": "Intra day steps data from fitbit device.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "timeInterval", "type": "int", "doc": "Chronological window size (s)." }, + { "name": "timezoneOffset", "type": "int", "doc": "Offset from UTC (s)." }, + { "name": "steps", "type": "int", "doc":"Steps taken in this period."} + ] +} diff --git a/commons/passive/fitbit/fitbit_sleep_classic.avsc b/commons/passive/fitbit/fitbit_sleep_classic.avsc deleted file mode 100644 index 9a0d3baa..00000000 --- a/commons/passive/fitbit/fitbit_sleep_classic.avsc +++ /dev/null @@ -1,25 +0,0 @@ -{ - "namespace": "org.radarcns.passive.fitbit", - "type": "record", - "name": "FitbitSleepClassic", - "doc": "Classic sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/", - "fields": [ - { - "name": "type", - "type": "string", - "doc": "Type of data coming from fitbit intraday API, (e.g. sleep classic)" - },{ - "name": "level", - "type": "string", - "doc": "Level of sleep, 'awake', 'restless', or 'asleep'" - },{ - "name": "dateTime", - "type": "string", - "doc": "Hours, minutes, seconds of start of interval" - },{ - "name": "seconds", - "type": "int", - "doc":"Duration at this sleep characteristic in secondse" - } - ] -} \ No newline at end of file diff --git a/commons/passive/fitbit/fitbit_sleep_pattern.avsc b/commons/passive/fitbit/fitbit_sleep_pattern.avsc new file mode 100644 index 00000000..7795812e --- /dev/null +++ b/commons/passive/fitbit/fitbit_sleep_pattern.avsc @@ -0,0 +1,13 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitSleepPattern", + "doc": "Classic sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "level", "type": "string", "doc": "Level of sleep, 'awake', 'restless', or 'asleep'." }, + { "name": "timezoneOffset", "type": "int", "doc": "Offset from UTC (s)." }, + { "name": "sleepDuration", "type": "int", "doc":"Duration at this sleep characteristic in seconds." } + ] +} diff --git a/commons/passive/fitbit/fitbit_sleep_stage.avsc b/commons/passive/fitbit/fitbit_sleep_stage.avsc new file mode 100644 index 00000000..2dc8d6e5 --- /dev/null +++ b/commons/passive/fitbit/fitbit_sleep_stage.avsc @@ -0,0 +1,13 @@ +{ + "namespace": "org.radarcns.passive.fitbit", + "type": "record", + "name": "FitbitSleepStage", + "doc": "Fitbit 'stages' sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "level", "type": "string", "doc": "Level of sleep, 'deep', 'light', 'rem', or 'awake'." }, + { "name": "timezoneOffset", "type": "int", "doc": "Offset from UTC (s)." }, + { "name": "sleepDuration", "type": "int", "doc":"Duration at this sleep characteristic in seconds." } + ] +} diff --git a/commons/passive/fitbit/fitbit_sleep_stages.avsc b/commons/passive/fitbit/fitbit_sleep_stages.avsc deleted file mode 100644 index 7358f05c..00000000 --- a/commons/passive/fitbit/fitbit_sleep_stages.avsc +++ /dev/null @@ -1,25 +0,0 @@ -{ - "namespace": "org.radarcns.passive.fitbit", - "type": "record", - "name": "FitbitSleepStages", - "doc": "Fitbit 'stages' sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/", - "fields": [ - { - "name": "type", - "type": "string", - "doc": "Type of data coming from fitbit intraday API, (e.g. sleep classic)" - },{ - "name": "level", - "type": "string", - "doc": "Level of sleep, 'deep', 'light', 'rem', or 'awake'" - },{ - "name": "dateTime", - "type": "string", - "doc": "Hours, minutes, seconds of start of interval" - },{ - "name": "seconds", - "type": "int", - "doc":"Duration at this sleep characteristic in secondse" - } - ] -} \ No newline at end of file From b767b8dc2c9bf9723077588c5cbea54a18aeb8df Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 12 Mar 2018 17:42:33 +0000 Subject: [PATCH 04/17] Remove null --- commons/active/thincit/thinc_it_pdq.avsc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/commons/active/thincit/thinc_it_pdq.avsc b/commons/active/thincit/thinc_it_pdq.avsc index f3bd1310..0a3ac060 100644 --- a/commons/active/thincit/thinc_it_pdq.avsc +++ b/commons/active/thincit/thinc_it_pdq.avsc @@ -7,7 +7,7 @@ { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the test is submitted to the subject." }, { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when the subject completes the test." }, { "name": "score", "type": "int", "doc": "THINC-it index score - 0 to 4000." }, - { "name": "questionOne", "type": ["null", + { "name": "questionOne", "type": { "type": "enum", "name": "ThincItPdqQuestion", @@ -19,11 +19,11 @@ "OFTEN", "VERYOFTEN" ] - }], "doc": "PDQ5 Question 1.", "default": null}, - { "name": "questionTwo", "type": ["null", "ThincItPdqQuestion"], "doc": "PDQ5 Question 2.", "default": null}, - { "name": "questionThree", "type": ["null", "ThincItPdqQuestion"], "doc": "PDQ5 Question 3.", "default": null}, - { "name": "questionFour", "type": ["null", "ThincItPdqQuestion"], "doc": "PDQ5 Question 4.", "default": null}, - { "name": "questionFive", "type": ["null", "ThincItPdqQuestion"], "doc": "PDQ5 Question 5.", "default": null}, + }, "doc": "PDQ5 Question 1."}, + { "name": "questionTwo", "type": "ThincItPdqQuestion", "doc": "PDQ5 Question 2."}, + { "name": "questionThree", "type": "ThincItPdqQuestion", "doc": "PDQ5 Question 3."}, + { "name": "questionFour", "type": "ThincItPdqQuestion", "doc": "PDQ5 Question 4."}, + { "name": "questionFive", "type": "ThincItPdqQuestion", "doc": "PDQ5 Question 5."}, { "name": "appVersion", "type": "int", "doc": "App version." } ] } From e169b5b56b4d476ecad627ae154df9eb0f1733b1 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Wed, 2 May 2018 10:59:13 +0100 Subject: [PATCH 05/17] Add output topic --- specifications/stream/android_phone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/specifications/stream/android_phone.yml b/specifications/stream/android_phone.yml index 910ff5a2..7c53c649 100644 --- a/specifications/stream/android_phone.yml +++ b/specifications/stream/android_phone.yml @@ -8,6 +8,7 @@ data: - input_topic: android_phone_usage_event value_schema: .passive.phone.PhoneUsageEvent - input_topic: android_phone_usage_event_output + output_topic: android_phone_usage_event_aggregated value_schema: .stream.aggregator.PhoneUsageAggregate windowed: true - input_topic: android_phone_battery_level From cf73af83b4f69835801cd8808297650466775f31 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 2 May 2018 15:45:23 +0100 Subject: [PATCH 06/17] Initial setup for Connector Specs --- .../main/java/org/radarcns/schema/Scope.java | 2 +- .../connector/ConnectorDataTopic.java | 7 ++++++ .../connector/ConnectorSource.java | 22 +++++++++++++++++++ .../specification/stream/StreamDataTopic.java | 7 ++++++ .../connector/radar-fitbit-connector.yml | 16 ++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java create mode 100644 java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java create mode 100644 specifications/connector/radar-fitbit-connector.yml diff --git a/java-sdk/radar-schemas-commons/src/main/java/org/radarcns/schema/Scope.java b/java-sdk/radar-schemas-commons/src/main/java/org/radarcns/schema/Scope.java index c0e03357..a21c2411 100644 --- a/java-sdk/radar-schemas-commons/src/main/java/org/radarcns/schema/Scope.java +++ b/java-sdk/radar-schemas-commons/src/main/java/org/radarcns/schema/Scope.java @@ -5,7 +5,7 @@ import java.util.Locale; public enum Scope { - ACTIVE, KAFKA, CATALOGUE, MONITOR, PASSIVE, STREAM; + ACTIVE, KAFKA, CATALOGUE, MONITOR, PASSIVE, STREAM, CONNECTOR; private final String lower; diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java new file mode 100644 index 00000000..3b9026a0 --- /dev/null +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java @@ -0,0 +1,7 @@ +package org.radarcns.schema.specification.connector; + +import org.radarcns.schema.specification.AppDataTopic; + +public class ConnectorDataTopic extends AppDataTopic { + +} diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java new file mode 100644 index 00000000..fe351aaf --- /dev/null +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java @@ -0,0 +1,22 @@ +package org.radarcns.schema.specification.connector; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.radarcns.schema.Scope; +import org.radarcns.schema.specification.AppSource; + +import java.util.List; + +public class ConnectorSource extends AppSource { + @JsonProperty + private List data; + + @Override + public List getData() { + return data; + } + + @Override + public Scope getScope() { + return Scope.CONNECTOR; + } +} diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java index 40e892b7..4a93f905 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java @@ -35,6 +35,9 @@ public class StreamDataTopic extends DataTopic { @JsonProperty("topic_base") private String topicBase; + @JsonProperty("output_topic") + private String outputTopic; + @JsonSetter @SuppressWarnings("PMD.UnusedPrivateMethod") private void setWindowed(boolean windowed) { @@ -59,6 +62,10 @@ private void setInputTopic(String inputTopic) { /** Get human readable output topic. */ public String getTopic() { + if (outputTopic != null) { + topicBase = outputTopic; + return topicBase; + } if (windowed) { return topicBase + "_"; } else if (super.getTopic() != null) { diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml new file mode 100644 index 00000000..d72bc915 --- /dev/null +++ b/specifications/connector/radar-fitbit-connector.yml @@ -0,0 +1,16 @@ +name: RADAR-FITBIT-CONNECTOR +doc: Spec for Radar fitbit connector. The schema should be registered via the connector. +master: .stream.phone.PhoneStreamMaster +data: + - schema_registration: false + topic: fitbit_intraday_heart_rate + value_schema: .passive.fitbit.FitbitIntradayHeartRate + - schema_registration: false + topic: fitbit_intraday_steps + value_schema: .passive.fitbit.FitbitIntradaySteps + - schema_registration: false + topic: fitbit_sleep_pattern + value_schema: .passive.fitbit.FitbitSleepPattern + - schema_registration: false + topic: fitbit_sleep_stage + value_schema: .passive.fitbit.FitbitSleepStage From e68c49e6d7c9ade3a3d3af548f8a89fd69fdc1d0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 3 May 2018 12:28:20 +0100 Subject: [PATCH 07/17] Addd validation and topic registration for connector --- .../fitbit/fitbit_intraday_heart_rate.avsc | 2 +- .../fitbit/fitbit_intraday_steps.avsc | 2 +- .../fitbit/fitbit_sleep_pattern.avsc | 2 +- .../fitbit/fitbit_sleep_stage.avsc | 2 +- .../config/pmd/ruleset.xml | 1 + .../org/radarcns/schema/CommandLineApp.java | 3 ++- .../schema/registration/SchemaRegistry.java | 2 +- .../schema/specification/SourceCatalogue.java | 18 ++++++++++++++++-- .../connector/ConnectorDataTopic.java | 7 ------- .../connector/ConnectorSource.java | 9 +++++---- .../specification/stream/StreamDataTopic.java | 10 ++++++++++ .../validation/SourceCatalogueValidation.java | 3 ++- .../connector/radar-fitbit-connector.yml | 19 +++++++++---------- 13 files changed, 50 insertions(+), 30 deletions(-) rename commons/{passive => connector}/fitbit/fitbit_intraday_heart_rate.avsc (92%) rename commons/{passive => connector}/fitbit/fitbit_intraday_steps.avsc (92%) rename commons/{passive => connector}/fitbit/fitbit_sleep_pattern.avsc (93%) rename commons/{passive => connector}/fitbit/fitbit_sleep_stage.avsc (93%) delete mode 100644 java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java diff --git a/commons/passive/fitbit/fitbit_intraday_heart_rate.avsc b/commons/connector/fitbit/fitbit_intraday_heart_rate.avsc similarity index 92% rename from commons/passive/fitbit/fitbit_intraday_heart_rate.avsc rename to commons/connector/fitbit/fitbit_intraday_heart_rate.avsc index 0ea9876f..a5ab056e 100644 --- a/commons/passive/fitbit/fitbit_intraday_heart_rate.avsc +++ b/commons/connector/fitbit/fitbit_intraday_heart_rate.avsc @@ -1,5 +1,5 @@ { - "namespace": "org.radarcns.passive.fitbit", + "namespace": "org.radarcns.connector.fitbit", "type": "record", "name": "FitbitIntradayHeartRate", "doc": "Intra day heart rate data from fitbit device.", diff --git a/commons/passive/fitbit/fitbit_intraday_steps.avsc b/commons/connector/fitbit/fitbit_intraday_steps.avsc similarity index 92% rename from commons/passive/fitbit/fitbit_intraday_steps.avsc rename to commons/connector/fitbit/fitbit_intraday_steps.avsc index 4bbbe044..7d93fd14 100644 --- a/commons/passive/fitbit/fitbit_intraday_steps.avsc +++ b/commons/connector/fitbit/fitbit_intraday_steps.avsc @@ -1,5 +1,5 @@ { - "namespace": "org.radarcns.passive.fitbit", + "namespace": "org.radarcns.connector.fitbit", "type": "record", "name": "FitbitIntradaySteps", "doc": "Intra day steps data from fitbit device.", diff --git a/commons/passive/fitbit/fitbit_sleep_pattern.avsc b/commons/connector/fitbit/fitbit_sleep_pattern.avsc similarity index 93% rename from commons/passive/fitbit/fitbit_sleep_pattern.avsc rename to commons/connector/fitbit/fitbit_sleep_pattern.avsc index 7795812e..cde19939 100644 --- a/commons/passive/fitbit/fitbit_sleep_pattern.avsc +++ b/commons/connector/fitbit/fitbit_sleep_pattern.avsc @@ -1,5 +1,5 @@ { - "namespace": "org.radarcns.passive.fitbit", + "namespace": "org.radarcns.connector.fitbit", "type": "record", "name": "FitbitSleepPattern", "doc": "Classic sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/.", diff --git a/commons/passive/fitbit/fitbit_sleep_stage.avsc b/commons/connector/fitbit/fitbit_sleep_stage.avsc similarity index 93% rename from commons/passive/fitbit/fitbit_sleep_stage.avsc rename to commons/connector/fitbit/fitbit_sleep_stage.avsc index 2dc8d6e5..f408f521 100644 --- a/commons/passive/fitbit/fitbit_sleep_stage.avsc +++ b/commons/connector/fitbit/fitbit_sleep_stage.avsc @@ -1,5 +1,5 @@ { - "namespace": "org.radarcns.passive.fitbit", + "namespace": "org.radarcns.connector.fitbit", "type": "record", "name": "FitbitSleepStage", "doc": "Fitbit 'stages' sleep data as defined at https://dev.fitbit.com/build/reference/web-api/sleep/.", diff --git a/java-sdk/radar-schemas-tools/config/pmd/ruleset.xml b/java-sdk/radar-schemas-tools/config/pmd/ruleset.xml index 8821c0b9..d888e0dd 100644 --- a/java-sdk/radar-schemas-tools/config/pmd/ruleset.xml +++ b/java-sdk/radar-schemas-tools/config/pmd/ruleset.xml @@ -10,6 +10,7 @@ + diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/CommandLineApp.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/CommandLineApp.java index 992076b4..be7cb7b7 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/CommandLineApp.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/CommandLineApp.java @@ -91,7 +91,8 @@ public List getRawTopics() { return Stream.of( catalogue.getPassiveSources(), catalogue.getActiveSources(), - catalogue.getMonitorSources()) + catalogue.getMonitorSources(), + catalogue.getConnectorSources()) .flatMap(map -> map.values().stream()) .flatMap(DataProducer::getTopicNames) .sorted() diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java index b0d68817..8e5e1913 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java @@ -73,7 +73,7 @@ public SchemaRegistry(String baseUrl) throws MalformedURLException { } /** - * Register all schemas in a source catalogue. Stream sources are ignored. + * Register all schemas in a source catalogue. Stream and connector sources are ignored. * @param catalogue schema catalogue to read schemas from * @return whether all schemas were successfully registered. */ diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java index 8a21aa6f..bc3301f4 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java @@ -24,6 +24,7 @@ import java.nio.file.InvalidPathException; import org.radarcns.schema.Scope; import org.radarcns.schema.specification.active.ActiveSource; +import org.radarcns.schema.specification.connector.ConnectorSource; import org.radarcns.schema.specification.stream.StreamGroup; import org.radarcns.schema.specification.monitor.MonitorSource; import org.radarcns.schema.specification.passive.PassiveSource; @@ -59,6 +60,7 @@ public class SourceCatalogue { private final Map> activeSources; private final Map monitorSources; private final Map passiveSources; + private final Map connectorSources; private final Map streamGroups; private final Set> sources; @@ -67,11 +69,13 @@ public class SourceCatalogue { SourceCatalogue(Map> activeSources, Map monitorSources, Map passiveSources, - Map streamGroups) { + Map streamGroups, + Map connectorSources) { this.activeSources = activeSources; this.monitorSources = monitorSources; this.passiveSources = passiveSources; this.streamGroups = streamGroups; + this.connectorSources = connectorSources; sources = new HashSet<>(); @@ -79,6 +83,7 @@ public class SourceCatalogue { sources.addAll(monitorSources.values()); sources.addAll(passiveSources.values()); sources.addAll(streamGroups.values()); + sources.addAll(connectorSources.values()); } /** @@ -107,7 +112,8 @@ public static SourceCatalogue load(Path root) throws IOException { initSources(mapper.readerFor(ActiveSource.class), specRoot, Scope.ACTIVE), initSources(mapper.readerFor(MonitorSource.class), specRoot, Scope.MONITOR), initSources(mapper.readerFor(PassiveSource.class), specRoot, Scope.PASSIVE), - initSources(mapper.readerFor(StreamGroup.class), specRoot, Scope.STREAM)); + initSources(mapper.readerFor(StreamGroup.class), specRoot, Scope.STREAM), + initSources(mapper.readerFor(ConnectorSource.class), specRoot, Scope.CONNECTOR)); } private static Map initSources(ObjectReader reader, Path root, Scope scope) @@ -207,6 +213,14 @@ public Stream getTopicNames() { .flatMap(DataProducer::getTopicNames); } + /** + * TODO. + * @return TODO + */ + public Map getConnectorSources() { + return connectorSources; + } + /** Get all topics in the catalogue. */ public Stream> getTopics() { return sources.stream() diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java deleted file mode 100644 index 3b9026a0..00000000 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorDataTopic.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.radarcns.schema.specification.connector; - -import org.radarcns.schema.specification.AppDataTopic; - -public class ConnectorDataTopic extends AppDataTopic { - -} diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java index fe351aaf..07a802c3 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java @@ -2,16 +2,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.radarcns.schema.Scope; -import org.radarcns.schema.specification.AppSource; +import org.radarcns.schema.specification.DataProducer; +import org.radarcns.schema.specification.DataTopic; import java.util.List; -public class ConnectorSource extends AppSource { +public class ConnectorSource extends DataProducer { @JsonProperty - private List data; + private List data; @Override - public List getData() { + public List getData() { return data; } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java index 4a93f905..af547093 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java @@ -60,6 +60,16 @@ private void setInputTopic(String inputTopic) { this.inputTopics.add(inputTopic); } + public String getOutputTopic() { + return outputTopic; + } + + @JsonSetter("output_topic") + @SuppressWarnings("PMD.UnusedPrivateMethod") + private void setOutputTopic(String outputTopic) { + this.outputTopic = outputTopic; + } + /** Get human readable output topic. */ public String getTopic() { if (outputTopic != null) { diff --git a/java-sdk/radar-schemas-tools/src/test/java/org/radarcns/schema/validation/SourceCatalogueValidation.java b/java-sdk/radar-schemas-tools/src/test/java/org/radarcns/schema/validation/SourceCatalogueValidation.java index 84cfc456..1ac20acb 100644 --- a/java-sdk/radar-schemas-tools/src/test/java/org/radarcns/schema/validation/SourceCatalogueValidation.java +++ b/java-sdk/radar-schemas-tools/src/test/java/org/radarcns/schema/validation/SourceCatalogueValidation.java @@ -54,7 +54,8 @@ public void validateTopics() { catalogue.getActiveSources(), catalogue.getMonitorSources(), catalogue.getPassiveSources(), - catalogue.getStreamGroups()) + catalogue.getStreamGroups(), + catalogue.getConnectorSources()) .flatMap(map -> map.values().stream()) .flatMap(DataProducer::getTopicNames) .sorted() diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index d72bc915..c7b4c997 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -1,16 +1,15 @@ name: RADAR-FITBIT-CONNECTOR -doc: Spec for Radar fitbit connector. The schema should be registered via the connector. -master: .stream.phone.PhoneStreamMaster +doc: Spec for Radar fitbit connector. The schemas should be registered via the connector. data: - - schema_registration: false + - doc: The intraday time series for heart rate. topic: fitbit_intraday_heart_rate - value_schema: .passive.fitbit.FitbitIntradayHeartRate - - schema_registration: false + value_schema: .connector.fitbit.FitbitIntradayHeartRate + - doc: The intraday time series for Steps. topic: fitbit_intraday_steps - value_schema: .passive.fitbit.FitbitIntradaySteps - - schema_registration: false + value_schema: .connector.fitbit.FitbitIntradaySteps + - doc: Sleep data with 60-second granularity. 'Sleep Pattern' levels include asleep, restless, and awake. topic: fitbit_sleep_pattern - value_schema: .passive.fitbit.FitbitSleepPattern - - schema_registration: false + value_schema: .connector.fitbit.FitbitSleepPattern + - doc: Sleep data with 30-second granularity. 'Sleep Stages' levels include deep, light, rem, and wake. topic: fitbit_sleep_stage - value_schema: .passive.fitbit.FitbitSleepStage + value_schema: .connector.fitbit.FitbitSleepStage From 036d6a83286c741f6b7bb98d9b2cb3230f61e974 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 9 May 2018 10:00:25 +0200 Subject: [PATCH 08/17] Schema registration update --- .../schema/registration/SchemaRegistry.java | 8 ++----- .../schema/specification/DataProducer.java | 15 +++++++++++++ .../connector/ConnectorSource.java | 9 ++++++++ .../specification/stream/StreamDataTopic.java | 21 +++---------------- .../specification/stream/StreamGroup.java | 9 ++++++++ .../connector/radar-fitbit-connector.yml | 3 ++- specifications/stream/android_phone.yml | 4 ++-- 7 files changed, 42 insertions(+), 27 deletions(-) diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java index 8e5e1913..e85f5ac8 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java @@ -44,7 +44,6 @@ import java.net.MalformedURLException; import java.util.Optional; import java.util.regex.Pattern; -import java.util.stream.Stream; import static org.radarcns.schema.CommandLineApp.matchTopic; @@ -78,11 +77,8 @@ public SchemaRegistry(String baseUrl) throws MalformedURLException { * @return whether all schemas were successfully registered. */ public boolean registerSchemas(SourceCatalogue catalogue) { - return Stream.of( - catalogue.getActiveSources(), - catalogue.getPassiveSources(), - catalogue.getMonitorSources()) - .flatMap(m -> m.values().stream()) + return catalogue.getSources().stream() + .filter(DataProducer::doRegisterSchema) .flatMap(DataProducer::getTopics) .allMatch(this::registerSchema); } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/DataProducer.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/DataProducer.java index 2484de24..5bbc4739 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/DataProducer.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/DataProducer.java @@ -13,6 +13,10 @@ import static org.radarcns.schema.util.Utils.applyOrEmpty; +/** + * A producer of data to Kafka, generally mapping to a source. + * @param type of data that is produced. + */ public abstract class DataProducer { @JsonProperty @NotBlank private String name; @@ -26,6 +30,14 @@ public abstract class DataProducer { @JsonProperty private List labels; + /** + * If true, register the schema during kafka initialization, otherwise, the producer should do + * that itself. The default is true, set in the constructor of subclasses to use a different + * default. + */ + @JsonProperty("register_schema") + protected boolean registerSchema = true; + public String getName() { return name; } @@ -56,6 +68,9 @@ public Stream getTopicNames() { return getData().stream().flatMap(applyOrEmpty(DataTopic::getTopics)); } + public boolean doRegisterSchema() { + return registerSchema; + } @Override public boolean equals(Object o) { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java index 07a802c3..5daf47ff 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/connector/ConnectorSource.java @@ -7,10 +7,19 @@ import java.util.List; +/** + * Data producer for third-party connectors. This data topic does not register schemas to the schema + * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set + * the {@code register_schema} property to {@code true}. + */ public class ConnectorSource extends DataProducer { @JsonProperty private List data; + public ConnectorSource() { + registerSchema = false; + } + @Override public List getData() { return data; diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java index af547093..6b47415a 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamDataTopic.java @@ -17,8 +17,10 @@ import org.radarcns.stream.TimeWindowMetadata; import org.radarcns.topic.AvroTopic; +/** + * Topic used for Kafka Streams. + */ public class StreamDataTopic extends DataTopic { - /** Whether the stream is a windowed stream with standard TimeWindow windows. */ @JsonProperty private boolean windowed = false; @@ -35,9 +37,6 @@ public class StreamDataTopic extends DataTopic { @JsonProperty("topic_base") private String topicBase; - @JsonProperty("output_topic") - private String outputTopic; - @JsonSetter @SuppressWarnings("PMD.UnusedPrivateMethod") private void setWindowed(boolean windowed) { @@ -60,22 +59,8 @@ private void setInputTopic(String inputTopic) { this.inputTopics.add(inputTopic); } - public String getOutputTopic() { - return outputTopic; - } - - @JsonSetter("output_topic") - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setOutputTopic(String outputTopic) { - this.outputTopic = outputTopic; - } - /** Get human readable output topic. */ public String getTopic() { - if (outputTopic != null) { - topicBase = outputTopic; - return topicBase; - } if (windowed) { return topicBase + "_"; } else if (super.getTopic() != null) { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamGroup.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamGroup.java index 4dfcdc8f..048c8924 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamGroup.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/stream/StreamGroup.java @@ -8,6 +8,11 @@ import java.util.List; import java.util.stream.Stream; +/** + * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry + * by default, since Kafka Streams will do that itself. To disable this, set the + * {@code register_schema} property to {@code true}. + */ public class StreamGroup extends DataProducer { @JsonProperty @NotEmpty private List data; @@ -15,6 +20,10 @@ public class StreamGroup extends DataProducer { @JsonProperty private String master; + public StreamGroup() { + registerSchema = false; + } + @Override public List getData() { return data; diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index c7b4c997..13c9e578 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -1,5 +1,6 @@ name: RADAR-FITBIT-CONNECTOR -doc: Spec for Radar fitbit connector. The schemas should be registered via the connector. +doc: Spec for Radar fitbit connector. +register_schema: true data: - doc: The intraday time series for heart rate. topic: fitbit_intraday_heart_rate diff --git a/specifications/stream/android_phone.yml b/specifications/stream/android_phone.yml index 7c53c649..7a0c627e 100644 --- a/specifications/stream/android_phone.yml +++ b/specifications/stream/android_phone.yml @@ -7,8 +7,8 @@ data: windowed: true - input_topic: android_phone_usage_event value_schema: .passive.phone.PhoneUsageEvent - - input_topic: android_phone_usage_event_output - output_topic: android_phone_usage_event_aggregated + - topic: android_phone_usage_event_aggregated + input_topic: android_phone_usage_event_output value_schema: .stream.aggregator.PhoneUsageAggregate windowed: true - input_topic: android_phone_battery_level From 028f3da07541f0ab9858c41010da364df042a9b8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 9 May 2018 10:02:30 +0200 Subject: [PATCH 09/17] Indentation fix --- .../java/org/radarcns/schema/specification/SourceCatalogue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java index bc3301f4..b1c870ec 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/specification/SourceCatalogue.java @@ -70,7 +70,7 @@ public class SourceCatalogue { Map monitorSources, Map passiveSources, Map streamGroups, - Map connectorSources) { + Map connectorSources) { this.activeSources = activeSources; this.monitorSources = monitorSources; this.passiveSources = passiveSources; From 7d43bc2856f57b5507a0c5fcb928246ed2d0c04d Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 11 May 2018 15:21:19 +0100 Subject: [PATCH 10/17] Add questionnaire completion log schema and specs --- .../questionnaire_completion_log.avsc | 11 +++++ .../schema/validation/rules/SchemaRules.java | 1 - specifications/active/aRMT-1.2.0.yml | 41 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 commons/active/questionnaire/questionnaire_completion_log.avsc create mode 100644 specifications/active/aRMT-1.2.0.yml diff --git a/commons/active/questionnaire/questionnaire_completion_log.avsc b/commons/active/questionnaire/questionnaire_completion_log.avsc new file mode 100644 index 00000000..06e96ac5 --- /dev/null +++ b/commons/active/questionnaire/questionnaire_completion_log.avsc @@ -0,0 +1,11 @@ +{ + "namespace": "org.radarcns.active.questionnaire", + "type": "record", + "name": "QuestionnaireCompletionLog", + "doc": "Schema for reporting the completion status of a questionnaire. This will help in calculating the compliance.", + "fields": [ + { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the questionnaire completion log is submitted." }, + { "name": "name", "type": "string", "doc": "Questionnaire name." }, + { "name": "isComplete", "type": "string", "doc": "It reports the questionnaire completion status. Could be true or false." } + ] +} diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java index d9bd97ff..90983ea3 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java @@ -74,7 +74,6 @@ default Validator validateRecord() { default Validator validateActiveSource() { return validateRecord() .and(validateTime() - .and(validateTimeCompleted()) .and(validateNotTimeReceived())); } diff --git a/specifications/active/aRMT-1.2.0.yml b/specifications/active/aRMT-1.2.0.yml new file mode 100644 index 00000000..285cb741 --- /dev/null +++ b/specifications/active/aRMT-1.2.0.yml @@ -0,0 +1,41 @@ +name: aRMT +vendor: RADAR +model: aRMT-App +version: 1.1.0 +assessment_type: QUESTIONNAIRE +doc: aRMT Questionnaires definition. Includes Personal Health Questionnaire Depression Scale (PHQ-8), Experience sampling method (ESM) and RSES and other data. +data: + - type: THINC-IT + topic: notification_thinc_it + value_schema: .active.notification.Notification + questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/thinc_it/thinc_it_armt.json + - type: ROMBERG_TEST + topic: task_romberg_test + value_schema: .active.task.Task + questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/romberg_test/romberg_test_armt.json + - type: 2MW_TEST + topic: task_2MW_test + value_schema: .active.task.Task + questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/2MW_test/2MW_test_armt.json + - type: TANDEM_WALKING_TEST + topic: task_tandem_walking_test + value_schema: .active.task.Task + questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/tandem_walking_test/tandem_walking_test_armt.json + - type: PHQ8 + topic: questionnaire_phq8 + value_schema: .active.questionnaire.Questionnaire + questionnaire_definition_url: https://raw.githubusercontent.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/master/questionnaires/phq8/phq8_armt.json + - type: ESM + topic: questionnaire_esm + value_schema: .active.questionnaire.Questionnaire + questionnaire_definition_url: https://raw.githubusercontent.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/master/questionnaires/esm/esm_armt.json + - type: AUDIO + topic: questionnaire_audio + value_schema: .active.opensmile.OpenSmile2AudioRecording + - type: RSES + topic: questionnaire_rses + value_schema: .active.questionnaire.Questionnaire + questionnaire_definition_url: https://raw.githubusercontent.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/master/questionnaires/rses/rses_armt.json + - type: COMPLETION_LOG + topic: questionnaire_completion_log + value_schema: .active.questionnaire.QuestionnaireCompletionLog From 48f7fb0a9f6bd5d88a293b0121f7931d7f8eaa59 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 11 May 2018 15:35:29 +0100 Subject: [PATCH 11/17] reefactor schema --- commons/active/questionnaire/questionnaire_completion_log.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/active/questionnaire/questionnaire_completion_log.avsc b/commons/active/questionnaire/questionnaire_completion_log.avsc index 06e96ac5..49fd233e 100644 --- a/commons/active/questionnaire/questionnaire_completion_log.avsc +++ b/commons/active/questionnaire/questionnaire_completion_log.avsc @@ -6,6 +6,6 @@ "fields": [ { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the questionnaire completion log is submitted." }, { "name": "name", "type": "string", "doc": "Questionnaire name." }, - { "name": "isComplete", "type": "string", "doc": "It reports the questionnaire completion status. Could be true or false." } + { "name": "completionPercentage", "type": "double", "doc": "Percentage of the questionnaire completed. 0 for not at all complete and 100 for full completion."} ] } From 24e286209d4dd01e02dda102828e6903b5ddea59 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 11 May 2018 15:46:34 +0100 Subject: [PATCH 12/17] Correct spec version --- specifications/active/aRMT-1.2.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifications/active/aRMT-1.2.0.yml b/specifications/active/aRMT-1.2.0.yml index 285cb741..7e8e6c9d 100644 --- a/specifications/active/aRMT-1.2.0.yml +++ b/specifications/active/aRMT-1.2.0.yml @@ -1,7 +1,7 @@ name: aRMT vendor: RADAR model: aRMT-App -version: 1.1.0 +version: 1.2.0 assessment_type: QUESTIONNAIRE doc: aRMT Questionnaires definition. Includes Personal Health Questionnaire Depression Scale (PHQ-8), Experience sampling method (ESM) and RSES and other data. data: From 77cfb1cc65e9c740d7327acd578e4b1b6bfe945c Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 14 May 2018 10:45:14 +0100 Subject: [PATCH 13/17] Add null --- commons/active/questionnaire/questionnaire_completion_log.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/active/questionnaire/questionnaire_completion_log.avsc b/commons/active/questionnaire/questionnaire_completion_log.avsc index 49fd233e..d5159876 100644 --- a/commons/active/questionnaire/questionnaire_completion_log.avsc +++ b/commons/active/questionnaire/questionnaire_completion_log.avsc @@ -6,6 +6,6 @@ "fields": [ { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the questionnaire completion log is submitted." }, { "name": "name", "type": "string", "doc": "Questionnaire name." }, - { "name": "completionPercentage", "type": "double", "doc": "Percentage of the questionnaire completed. 0 for not at all complete and 100 for full completion."} + { "name": "completionPercentage", "type": [ "null", "double"], "doc": "Percentage of the questionnaire completed. 0 for not at all complete and 100 for full completion. Null if no completion value possible" , "default": null } ] } From 8e4e7ef2508018c28462644a6203e0a8f1ee932e Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 14 May 2018 10:49:31 +0100 Subject: [PATCH 14/17] validation fix --- commons/active/questionnaire/questionnaire_completion_log.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/active/questionnaire/questionnaire_completion_log.avsc b/commons/active/questionnaire/questionnaire_completion_log.avsc index d5159876..092ac250 100644 --- a/commons/active/questionnaire/questionnaire_completion_log.avsc +++ b/commons/active/questionnaire/questionnaire_completion_log.avsc @@ -6,6 +6,6 @@ "fields": [ { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the questionnaire completion log is submitted." }, { "name": "name", "type": "string", "doc": "Questionnaire name." }, - { "name": "completionPercentage", "type": [ "null", "double"], "doc": "Percentage of the questionnaire completed. 0 for not at all complete and 100 for full completion. Null if no completion value possible" , "default": null } + { "name": "completionPercentage", "type": [ "null", "double"], "doc": "Percentage of the questionnaire completed. 0 for not at all complete and 100 for full completion. Null if no completion value possible." , "default": null } ] } From 8860dec55ce9e346cdc74c50e11a6c9c6aac63ef Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 14 May 2018 16:28:14 +0100 Subject: [PATCH 15/17] Move to monitor scope --- .../questionnaire_completion_log.avsc | 2 +- .../schema/validation/rules/SchemaRules.java | 31 ++++++++++++++----- specifications/active/aRMT-1.2.0.yml | 2 +- 3 files changed, 26 insertions(+), 9 deletions(-) rename commons/{active => monitor}/questionnaire/questionnaire_completion_log.avsc (92%) diff --git a/commons/active/questionnaire/questionnaire_completion_log.avsc b/commons/monitor/questionnaire/questionnaire_completion_log.avsc similarity index 92% rename from commons/active/questionnaire/questionnaire_completion_log.avsc rename to commons/monitor/questionnaire/questionnaire_completion_log.avsc index 092ac250..5c9a7772 100644 --- a/commons/active/questionnaire/questionnaire_completion_log.avsc +++ b/commons/monitor/questionnaire/questionnaire_completion_log.avsc @@ -1,5 +1,5 @@ { - "namespace": "org.radarcns.active.questionnaire", + "namespace": "org.radarcns.monitor.questionnaire", "type": "record", "name": "QuestionnaireCompletionLog", "doc": "Schema for reporting the completion status of a questionnaire. This will help in calculating the compliance.", diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java index 90983ea3..818bf8da 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/validation/rules/SchemaRules.java @@ -9,19 +9,29 @@ public interface SchemaRules { SchemaFieldRules getFieldRules(); - /** Checks that schemas are unique compared to already validated schemas. */ + /** + * Checks that schemas are unique compared to already validated schemas. + */ Validator validateUniqueness(); - /** Checks schema namespace format. */ + /** + * Checks schema namespace format. + */ Validator validateNameSpace(); - /** Checks schema name format. */ + /** + * Checks schema name format. + */ Validator validateName(); - /** Checks schema documentation presence and format. */ + /** + * Checks schema documentation presence and format. + */ Validator validateSchemaDocumentation(); - /** Checks that the symbols of enums have the required format. */ + /** + * Checks that the symbols of enums have the required format. + */ Validator validateSymbols(); /** @@ -49,7 +59,9 @@ public interface SchemaRules { */ Validator validateNotTimeReceived(); - /** Validate an enum. */ + /** + * Validate an enum. + */ default Validator validateEnum() { return validateUniqueness() .and(validateNameSpace()) @@ -58,7 +70,9 @@ default Validator validateEnum() { .and(validateName()); } - /** Validate a record that is defined inline. */ + /** + * Validate a record that is defined inline. + */ default Validator validateRecord() { return validateUniqueness() .and(validateNameSpace()) @@ -69,16 +83,19 @@ default Validator validateRecord() { /** * Validates record schemas of an active source. + * * @return TODO */ default Validator validateActiveSource() { return validateRecord() .and(validateTime() + .and(validateTimeCompleted()) .and(validateNotTimeReceived())); } /** * Validates schemas of monitor sources. + * * @return TODO */ default Validator validateMonitor() { diff --git a/specifications/active/aRMT-1.2.0.yml b/specifications/active/aRMT-1.2.0.yml index 7e8e6c9d..d150726a 100644 --- a/specifications/active/aRMT-1.2.0.yml +++ b/specifications/active/aRMT-1.2.0.yml @@ -38,4 +38,4 @@ data: questionnaire_definition_url: https://raw.githubusercontent.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/master/questionnaires/rses/rses_armt.json - type: COMPLETION_LOG topic: questionnaire_completion_log - value_schema: .active.questionnaire.QuestionnaireCompletionLog + value_schema: .monitor.questionnaire.QuestionnaireCompletionLog From 3c2c35de09e98442b50c1d48d35cba9d15b6a3a5 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 14 May 2018 18:07:27 +0100 Subject: [PATCH 16/17] Don't register schema for the fitbit connector --- specifications/connector/radar-fitbit-connector.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index 13c9e578..ff08e6a3 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -1,6 +1,5 @@ name: RADAR-FITBIT-CONNECTOR -doc: Spec for Radar fitbit connector. -register_schema: true +doc: Spec for Radar fitbit connector. Schemas should be registered in the connector. data: - doc: The intraday time series for heart rate. topic: fitbit_intraday_heart_rate From 2e1a5fdcf3c4d6b8f602a6ebc0e74c1bf02015c4 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Tue, 15 May 2018 11:43:44 +0100 Subject: [PATCH 17/17] Bump version --- java-sdk/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/build.gradle b/java-sdk/build.gradle index 0823b306..81cdc3c7 100644 --- a/java-sdk/build.gradle +++ b/java-sdk/build.gradle @@ -17,7 +17,7 @@ subprojects { apply plugin: 'idea' // Configuration - version = '0.3.3-SNAPSHOT' + version = '0.3.3' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Schemas'