diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d3cc21fe --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +.gradle +.idea +out +build +*/out +*/src/test +*/build +*.iml +.gradletasknamecache diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..46c50c4c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 +continuation_indent_size = 4 + +[*.gradle] +indent_size = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..fdb42386 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.gradletasknamecache +build/ +.idea/ +*.iml +out/ +.gradle/ +docker/users +docker/source-fitbit.properties diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..458569bb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +# Copyright 2018 The Hyve +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM openjdk:8-alpine as builder + +RUN mkdir /code +WORKDIR /code + +ENV GRADLE_OPTS -Dorg.gradle.daemon=false + +COPY ./gradle/wrapper /code/gradle/wrapper +COPY ./gradlew /code/ +RUN ./gradlew --version + +COPY ./build.gradle ./settings.gradle /code/ +COPY kafka-connect-rest-source/build.gradle /code/kafka-connect-rest-source/ + +RUN ./gradlew downloadDependencies copyDependencies + +COPY kafka-connect-fitbit-source/build.gradle /code/kafka-connect-fitbit-source/ + +RUN ./gradlew downloadDependencies copyDependencies + +COPY ./kafka-connect-rest-source/src/ /code/kafka-connect-rest-source/src + +RUN ./gradlew jar + +COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src + +RUN ./gradlew jar + +FROM confluentinc/cp-kafka-connect-base:5.0.0 + +MAINTAINER Joris Borgdorff + +LABEL description="Kafka MongoDB Sink Connector" + +ENV CONNECT_PLUGIN_PATH /usr/share/java/kafka-connect/plugins + +# To isolate the classpath from the plugin path as recommended +COPY --from=builder /code/kafka-connect-rest-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-rest-source/ +COPY --from=builder /code/kafka-connect-fitbit-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-fitbit-source/ + +COPY --from=builder /code/kafka-connect-rest-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-rest-source/ +COPY --from=builder /code/kafka-connect-rest-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-fitbit-source/ +COPY --from=builder /code/kafka-connect-fitbit-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-fitbit-source/ + +# Load topics validator +COPY ./docker/kafka-wait /usr/bin/kafka-wait + +# Load modified launcher +COPY ./docker/launch /etc/confluent/docker/launch diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 47a5a497..419ff954 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,45 @@ -Kafka Connect REST Connector -=== +# Kafka Connect REST Source and Fitbit Source +This project contains a Kafka Connect source connector for a general REST API, and one for +Fitbit in particular. The documentation of the Kafka Connect REST source still needs to be done. +## Fitbit source connector -Building and running Fitbit example ---- -Begin Fitbit instance +### Installation - docker exec -it spring_connect_1 bash -c \ - "kafka-topics --zookeeper zookeeper \ - --topic restSourceDestinationTopic --create \ - --replication-factor 1 --partitions 1" +This repository relies on a recent version of docker and docker-compose as well as an installation +of Java 8 or later. -Pull history of activity +### Usage - curl -X POST \ - -H 'Host: connect.example.com' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - http://localhost:8083/connectors -d @config/activity_history.json +First, [register a Fitbit App](https://dev.fitbit.com/apps) with Fitbit. It should be either a +server app, for multiple users, or a personal app for a single user. With the server app, you need +to [request access to intraday API data](https://dev.fitbit.com/build/reference/web-api/help/). -Change CONNECT_VALUE_CONVERTER in the docker-compose.yml to org.apache.kafka.connect.storage.StringConverter if you don't want to use Avro. +For every Fitbit user you want access to, copy `docker/fitbit-user.yml.template` to a file in +`docker/users/`. Get an access token and refresh token for the user using for example the +[Fitbit OAuth 2.0 tutorial page](https://dev.fitbit.com/apps/oauthinteractivetutorial). - docker exec -it spring_connect_1 bash -c \ - "kafka-avro-console-consumer --bootstrap-server kafka:9092 \ - --topic restSourceDestinationTopic --from-beginning \ - --property schema.registry.url=http://schema_registry:8081/" +Copy `docker/source-fitbit.properties.template` to `docker/source-fitbit.properties` and enter +your Fitbit App client ID and client secret. - docker logs -f spring_webservice_1 +Now you can run a full Kafka stack using - docker-compose down - cd ../.. +```shell +docker-compose up -d --build +``` -Change CONNECT_VALUE_CONVERTER in the docker-compose.yml -to org.apache.kafka.connect.storage.StringConverter if you don't want to use Avro. +Inspect the progress with `docker-compose logs -f radar-fitbit-connector`. To inspect data +that is coming out of the requests, run - docker exec -it spring_connect_1 bash -c \ - "kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic restSourceDestinationTopic --from-beginning" +```shell +docker-compose exec schema-registry-1 kafka-avro-console-consumer \ + --bootstrap-server kafka-1:9092,kafka-2:9092,kafka-3:9092 \ + --from-beginning \ + --topic connect_fitbit_intraday_heart_rate +``` +## Contributing + +Code should be formatted using the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html). +If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-base/RADAR-REST-Connector/issues), and please make a pull request. diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000..5c8fe293 --- /dev/null +++ b/build.gradle @@ -0,0 +1,43 @@ +description = 'kafka-connect-rest-source' + +subprojects { + ext { + kafkaVersion = '2.0.0' + confluentVersion = '5.0.0' + jacksonVersion = '2.8.9' + } + + apply plugin: 'java' + apply plugin: 'java-library' + + group = 'org.radarcns' + version = '0.1-SNAPSHOT' + + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + + repositories { + mavenCentral() + maven { url "http://packages.confluent.io/maven/" } + maven { url "http://repo.maven.apache.org/maven2" } + jcenter() + maven { url 'http://oss.jfrog.org/artifactory/oss-snapshot-local/' } + } +} + +wrapper { + gradleVersion '4.9' +} + +evaluationDependsOnChildren() + +task downloadDependencies { + subprojects.collect { + it.configurations.runtimeClasspath.files + it.configurations.compileClasspath.files + } + + doLast { + println 'Downloaded REST code dependencies' + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..e50d7cfa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,185 @@ +--- +version: '2.4' + +volumes: + fitbit-logs: {} + +services: + #---------------------------------------------------------------------------# + # Zookeeper Cluster # + #---------------------------------------------------------------------------# + zookeeper-1: + image: confluentinc/cp-zookeeper:5.0.0 + environment: + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 1000 + ZOOKEEPER_INIT_LIMIT: 5 + ZOOKEEPER_SYNC_LIMIT: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + zookeeper-2: + image: confluentinc/cp-zookeeper:5.0.0 + environment: + ZOOKEEPER_SERVER_ID: 2 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 1000 + ZOOKEEPER_INIT_LIMIT: 5 + ZOOKEEPER_SYNC_LIMIT: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + zookeeper-3: + image: confluentinc/cp-zookeeper:5.0.0 + environment: + ZOOKEEPER_SERVER_ID: 3 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 1000 + ZOOKEEPER_INIT_LIMIT: 5 + ZOOKEEPER_SYNC_LIMIT: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + #---------------------------------------------------------------------------# + # Kafka Cluster # + #---------------------------------------------------------------------------# + kafka-1: + image: confluentinc/cp-kafka:5.0.0 + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "2.0" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "2.0" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + + kafka-2: + image: confluentinc/cp-kafka:5.0.0 + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "2.0" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "2.0" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + + kafka-3: + image: confluentinc/cp-kafka:5.0.0 + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "2.0" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "2.0" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + + #---------------------------------------------------------------------------# + # Schema Registry # + #---------------------------------------------------------------------------# + schema-registry-1: + image: confluentinc/cp-schema-registry:5.0.0 + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + restart: always + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + SCHEMA_REGISTRY_HOST_NAME: schema-registry-1 + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + SCHEMA_REGISTRY_AVRO_COMPATIBILITY_LEVEL: none + KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + + #---------------------------------------------------------------------------# + # REST proxy # + #---------------------------------------------------------------------------# + rest-proxy-1: + image: confluentinc/cp-kafka-rest:5.0.0 + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + ports: + - "8082:8082" + environment: + KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + KAFKA_REST_LISTENERS: http://0.0.0.0:8082 + KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry-1:8081 + KAFKA_REST_HOST_NAME: rest-proxy-1 + KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + KAFKA_AUTO_COMMIT_INTERVAL_MS: 300 + KAFKA_REST_COMPRESSION_TYPE: lz4 + + #---------------------------------------------------------------------------# + # RADAR Fitbit connector # + #---------------------------------------------------------------------------# + radar-fitbit-connector: + build: . + image: radarbase/radar-connect-fitbit-source + restart: on-failure + volumes: + - ./docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties + - ./docker/users:/var/lib/kafka-connect-fitbit-source/users + - fitbit-logs:/var/lib/kafka-connect-fitbit-source/logs + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-fitbit-source/logs/connect.offsets" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-fitbit-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "source-fitbit" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" diff --git a/docker/fitbit-user.yml.template b/docker/fitbit-user.yml.template new file mode 100644 index 00000000..56f4eeea --- /dev/null +++ b/docker/fitbit-user.yml.template @@ -0,0 +1,24 @@ +--- +# Unique user key +id: test +# Project ID to be used in org.radarcns.kafka.ObservationKey record keys +projectId: radar-test +# User ID to be used in org.radarcns.kafka.ObservationKey record keys +userId: test +# Source ID to be used in org.radarcns.kafka.ObservationKey record keys +sourceId: charge-2 +# Date from when to collect data. +startDate: 2018-08-06T00:00:00Z +# Date until when to collect data. +endDate: 2019-01-01T00:00:00Z +# Fitbit user ID as returned by the Fitbit authentication procedure +fitbitUserId: ? +oauth2: + # Fitbit OAuth 2.0 access token as returned by the Fitbit authentication procedure + accessToken: ? + # Fitbit OAuth 2.0 refresh token as returned by the Fitbit authentication procedure + refreshToken: ? + # Optional expiry time of the access token. If absent, it will be estimated to one hour + # when the source connector starts. When an authentication error occurs, a new access token will + # be fetched regardless of the value in this field. + #expiresAt: 2018-08-06T00:00:00Z diff --git a/docker/kafka-wait b/docker/kafka-wait new file mode 100755 index 00000000..a971cd30 --- /dev/null +++ b/docker/kafka-wait @@ -0,0 +1,65 @@ +#!/bin/bash + +# Check if variables exist +if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then + echo "CONNECT_ZOOKEEPER_CONNECT is not defined" + exit 2 +fi + +if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then + echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" + exit 4 +fi + +KAFKA_BROKERS=${KAFKA_BROKERS:-3} + +max_timeout=32 + +tries=10 +timeout=1 +while true; do + ZOOKEEPER_CHECK=$(zookeeper-shell ${CONNECT_ZOOKEEPER_CONNECT} <<< "ls /brokers/ids" | tail -2 | head -1) + echo "Zookeeper response: ${ZOOKEEPER_CHECK}" + # ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" + ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" + + IFS=',' read -r -a array <<< ${ZOOKEEPER_CHECK} + LENGTH=${#array[@]} + if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then + echo "Kafka brokers available." + break + fi + + tries=$((tries - 1)) + if [ ${tries} -eq 0 ]; then + echo "FAILED: KAFKA BROKERs NOT READY." + exit 5 + fi + echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." + sleep ${timeout} + if [ ${timeout} -lt ${max_timeout} ]; then + timeout=$((timeout * 2)) + fi +done + +tries=10 +timeout=1 +while true; do + if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then + echo "Schema registry available." + break + fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY." + exit 6 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep ${timeout} + if [ ${timeout} -lt ${max_timeout} ]; then + timeout=$((timeout * 2)) + fi +done + + +echo "Kafka is available. Ready to go!" diff --git a/docker/launch b/docker/launch new file mode 100755 index 00000000..a9ba474b --- /dev/null +++ b/docker/launch @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi + +# Busy waiting loop that waits untill all topic are available +echo "===> Wait for infrastructure ..." +kafka-wait +radar_check=$? +if [ "$radar_check" -ne 0 ]; then + exit ${radar_check} +fi + +echo "===> Launching ${COMPONENT} ..." +# Add our jar to the classpath so that the custom classes can be loaded first. +# And this also makes sure that the CLASSPATH does not start with ":/etc/..." +# other jars are loaded via the plugin path +export CLASSPATH="/etc/${COMPONENT}/kafka-connect-mongodb-sink/*" + +# execute connector in standalone mode +exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties $(find /etc/"${COMPONENT}"/ -type f -name "${CONNECTOR_PROPERTY_FILE_PREFIX}*.properties") diff --git a/docker/source-fitbit.properties.template b/docker/source-fitbit.properties.template new file mode 100644 index 00000000..1b5ed826 --- /dev/null +++ b/docker/source-fitbit.properties.template @@ -0,0 +1,8 @@ +name=radar-fitbit-source +connector.class=org.radarbase.connect.rest.fitbit.FitbitSourceConnector +tasks.max=4 +rest.source.base.url=https://api.fitbit.com +rest.source.poll.interval.ms=5000 +rest.source.request.generator.class=org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator +fitbit.api.client=? +fitbit.api.secret=? diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..0d4a9516 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a95009c3 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..cccdd3d5 --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..e95643d6 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/kafka-connect-fitbit-source/build.gradle b/kafka-connect-fitbit-source/build.gradle new file mode 100644 index 00000000..2ca11b87 --- /dev/null +++ b/kafka-connect-fitbit-source/build.gradle @@ -0,0 +1,25 @@ +dependencies { + api project(':kafka-connect-rest-source') + api group: 'io.confluent', name: 'kafka-connect-avro-converter', version: confluentVersion + api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.3.7-SNAPSHOT' + + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion + implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion + + // Included in connector runtime + compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion + compileOnly group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.2.0' + testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.2.0' + testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25' +} + +task copyDependencies(type: Copy) { + from configurations.runtimeClasspath.files + into "${buildDir}/third-party/" +} + +test { + useJUnitPlatform() +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java new file mode 100644 index 00000000..4434fd68 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java @@ -0,0 +1,271 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit; + +import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; + +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import okhttp3.Headers; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.errors.ConnectException; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.config.ValidClass; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.radarbase.connect.rest.fitbit.user.YamlFitbitUserRepository; + +public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig { + public static final String FITBIT_USERS_CONFIG = "fitbit.users"; + private static final String FITBIT_USERS_DOC = + "Fitbit users with syntax fitbitUserName:refreshToken:projectId:userName:startDate:endDate" + + "separated by commas."; + private static final String FITBIT_USERS_DISPLAY = "Fitbit users"; + + public static final String FITBIT_API_CLIENT_CONFIG = "fitbit.api.client"; + private static final String FITBIT_API_CLIENT_DOC = + "Client ID for the Fitbit API"; + private static final String FITBIT_API_CLIENT_DISPLAY = "Fitbit API client ID"; + + public static final String FITBIT_API_SECRET_CONFIG = "fitbit.api.secret"; + private static final String FITBIT_API_SECRET_DOC = "Secret for the Fitbit API client set in fitbit.api.client."; + private static final String FITBIT_API_SECRET_DISPLAY = "Fitbit API client secret"; + + public static final String FITBIT_USER_REPOSITORY_CONFIG = "fitbit.user.repository.class"; + private static final String FITBIT_USER_REPOSITORY_DOC = "Class for managing users and authentication."; + private static final String FITBIT_USER_REPOSITORY_DISPLAY = "User repository class"; + + public static final String FITBIT_USER_CREDENTIALS_DIR_CONFIG = "fitbit.user.dir"; + private static final String FITBIT_USER_CREDENTIALS_DIR_DOC = "Directory containing Fitbit user information and credentials. Only used if a file-based user repository is configured."; + private static final String FITBIT_USER_CREDENTIALS_DIR_DISPLAY = "User directory"; + private static final String FITBIT_USER_CREDENTIALS_DIR_DEFAULT = "/var/lib/kafka-connect-fitbit-source/users"; + + private static final String FITBIT_INTRADAY_STEPS_TOPIC_CONFIG = "fitbit.intraday.steps.topic"; + private static final String FITBIT_INTRADAY_STEPS_TOPIC_DOC = "Topic for Fitbit intraday steps"; + private static final String FITBIT_INTRADAY_STEPS_TOPIC_DISPLAY = "Intraday steps topic"; + private static final String FITBIT_INTRADAY_STEPS_TOPIC_DEFAULT = "connect_fitbit_intraday_steps"; + + private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_CONFIG = "fitbit.intraday.heart.rate.topic"; + private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_DOC = "Topic for Fitbit intraday heart_rate"; + private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_DISPLAY = "Intraday heartrate topic"; + private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_DEFAULT = "connect_fitbit_intraday_heart_rate"; + + private static final String FITBIT_SLEEP_STAGES_TOPIC_CONFIG = "fitbit.sleep.stages.topic"; + private static final String FITBIT_SLEEP_STAGES_TOPIC_DOC = "Topic for Fitbit sleep stages"; + private static final String FITBIT_SLEEP_STAGES_TOPIC_DEFAULT = "connect_fitbit_sleep_stages"; + private static final String FITBIT_SLEEP_STAGES_TOPIC_DISPLAY = "Sleep stages topic"; + + private static final String FITBIT_SLEEP_CLASSIC_TOPIC_CONFIG = "fitbit.sleep.classic.topic"; + private static final String FITBIT_SLEEP_CLASSIC_TOPIC_DOC = "Topic for Fitbit sleep classic data"; + private static final String FITBIT_SLEEP_CLASSIC_TOPIC_DEFAULT = "connect_fitbit_sleep_classic"; + private static final String FITBIT_SLEEP_CLASSIC_TOPIC_DISPLAY = "Classic sleep topic"; + + private static final String FITBIT_TIME_ZONE_TOPIC_CONFIG = "fitbit.time.zone.topic"; + private static final String FITBIT_TIME_ZONE_TOPIC_DOC = "Topic for Fitbit profile timezone"; + private static final String FITBIT_TIME_ZONE_TOPIC_DEFAULT = "connect_fitbit_time_zone"; + private static final String FITBIT_TIME_ZONE_TOPIC_DISPLAY = "Timezone topic"; + + + private final FitbitUserRepository fitbitUserRepository; + private final Headers clientCredentials; + + @SuppressWarnings("unchecked") + public FitbitRestSourceConnectorConfig(ConfigDef config, Map parsedConfig) { + super(config, parsedConfig); + + try { + fitbitUserRepository = ((Class) + getClass(FITBIT_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException + | InvocationTargetException | NoSuchMethodException e) { + throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e); + } + + String credentialString = getFitbitClient() + ":" + getFitbitClientSecret(); + String credentialsBase64 = Base64.getEncoder().encodeToString( + credentialString.getBytes(StandardCharsets.UTF_8)); + this.clientCredentials = Headers.of("Authorization", "Basic " + credentialsBase64); + } + + public FitbitRestSourceConnectorConfig(Map parsedConfig) { + this(FitbitRestSourceConnectorConfig.conf(), parsedConfig); + } + + public static ConfigDef conf() { + int orderInGroup = 0; + String group = "Fitbit"; + return RestSourceConnectorConfig.conf() + + .define(FITBIT_USERS_CONFIG, + ConfigDef.Type.LIST, + Collections.emptyList(), + ConfigDef.Importance.HIGH, + FITBIT_USERS_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_USERS_DISPLAY) + + .define(FITBIT_API_CLIENT_CONFIG, + ConfigDef.Type.STRING, + NO_DEFAULT_VALUE, + new ConfigDef.NonEmptyString(), + ConfigDef.Importance.HIGH, + FITBIT_API_CLIENT_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_API_CLIENT_DISPLAY) + + .define(FITBIT_API_SECRET_CONFIG, + ConfigDef.Type.PASSWORD, + NO_DEFAULT_VALUE, + ConfigDef.Importance.HIGH, + FITBIT_API_SECRET_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_API_SECRET_DISPLAY) + + .define(FITBIT_USER_REPOSITORY_CONFIG, + ConfigDef.Type.CLASS, + YamlFitbitUserRepository.class, + ValidClass.isSubclassOf(FitbitUserRepository.class), + ConfigDef.Importance.MEDIUM, + FITBIT_USER_REPOSITORY_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_USER_REPOSITORY_DISPLAY) + + .define(FITBIT_USER_CREDENTIALS_DIR_CONFIG, + ConfigDef.Type.STRING, + FITBIT_USER_CREDENTIALS_DIR_DEFAULT, + ConfigDef.Importance.LOW, + FITBIT_USER_CREDENTIALS_DIR_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_USER_CREDENTIALS_DIR_DISPLAY) + + .define(FITBIT_INTRADAY_STEPS_TOPIC_CONFIG, + ConfigDef.Type.STRING, + FITBIT_INTRADAY_STEPS_TOPIC_DEFAULT, + new ConfigDef.NonEmptyStringWithoutControlChars(), + ConfigDef.Importance.LOW, + FITBIT_INTRADAY_STEPS_TOPIC_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_INTRADAY_STEPS_TOPIC_DISPLAY) + + .define(FITBIT_INTRADAY_HEART_RATE_TOPIC_CONFIG, + ConfigDef.Type.STRING, + FITBIT_INTRADAY_HEART_RATE_TOPIC_DEFAULT, + new ConfigDef.NonEmptyStringWithoutControlChars(), + ConfigDef.Importance.LOW, + FITBIT_INTRADAY_HEART_RATE_TOPIC_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_INTRADAY_HEART_RATE_TOPIC_DISPLAY) + + .define(FITBIT_SLEEP_STAGES_TOPIC_CONFIG, + ConfigDef.Type.STRING, + FITBIT_SLEEP_STAGES_TOPIC_DEFAULT, + new ConfigDef.NonEmptyStringWithoutControlChars(), + ConfigDef.Importance.LOW, + FITBIT_SLEEP_STAGES_TOPIC_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_SLEEP_STAGES_TOPIC_DISPLAY) + + .define(FITBIT_SLEEP_CLASSIC_TOPIC_CONFIG, + ConfigDef.Type.STRING, + FITBIT_SLEEP_CLASSIC_TOPIC_DEFAULT, + new ConfigDef.NonEmptyStringWithoutControlChars(), + ConfigDef.Importance.LOW, + FITBIT_SLEEP_CLASSIC_TOPIC_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_SLEEP_CLASSIC_TOPIC_DISPLAY) + + .define(FITBIT_TIME_ZONE_TOPIC_CONFIG, + ConfigDef.Type.STRING, + FITBIT_TIME_ZONE_TOPIC_DEFAULT, + new ConfigDef.NonEmptyStringWithoutControlChars(), + ConfigDef.Importance.LOW, + FITBIT_TIME_ZONE_TOPIC_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + FITBIT_TIME_ZONE_TOPIC_DISPLAY) + ; + } + + public List getFitbitUsers() { + return getList(FITBIT_USERS_CONFIG); + } + + public String getFitbitClient() { + return getString(FITBIT_API_CLIENT_CONFIG); + } + + public String getFitbitClientSecret() { + return getPassword(FITBIT_API_SECRET_CONFIG).value(); + } + + public FitbitUserRepository getFitbitUserRepository() { + fitbitUserRepository.initialize(this); + return fitbitUserRepository; + } + + public String getFitbitIntradayStepsTopic() { + return getString(FITBIT_INTRADAY_STEPS_TOPIC_CONFIG); + } + + public String getFitbitSleepStagesTopic() { + return getString(FITBIT_SLEEP_STAGES_TOPIC_CONFIG); + } + + public String getFitbitTimeZoneTopic() { + return getString(FITBIT_TIME_ZONE_TOPIC_CONFIG); + } + + public String getFitbitIntradayHeartRateTopic() { + return getString(FITBIT_INTRADAY_HEART_RATE_TOPIC_CONFIG); + } + public String getFitbitSleepClassicTopic() { + return getString(FITBIT_SLEEP_CLASSIC_TOPIC_CONFIG); + } + + public Path getFitbitUserCredentialsPath() { + return Paths.get(getString(FITBIT_USER_CREDENTIALS_DIR_CONFIG)); + } + + public Headers getClientCredentials() { + return clientCredentials; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java new file mode 100644 index 00000000..86c94af8 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitSourceConnector.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit; + +import static org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig.FITBIT_USERS_CONFIG; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; +import org.radarbase.connect.rest.AbstractRestSourceConnector; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; + +public class FitbitSourceConnector extends AbstractRestSourceConnector { + @Override + public FitbitRestSourceConnectorConfig getConfig(Map conf) { + return new FitbitRestSourceConnectorConfig(conf); + } + + @Override + public ConfigDef config() { + return FitbitRestSourceConnectorConfig.conf(); + } + + @Override + public List> taskConfigs(int maxTasks) { + Map baseConfig = config.originalsStrings(); + FitbitRestSourceConnectorConfig fitbitConfig = getConfig(baseConfig); + // Divide the users over tasks + try { + return fitbitConfig.getFitbitUserRepository().stream() + .map(FitbitUser::getId) + // group users based on their hashCode + // in principle this allows for more efficient reconfigurations for a fixed number of tasks, + // since that allows existing tasks to only handle small modifications users to handle. + .collect(Collectors.groupingBy( + u -> Math.abs(u.hashCode()) % maxTasks, + Collectors.joining(","))) + .values().stream() + .map(u -> { + Map config = new HashMap<>(baseConfig); + config.put(FITBIT_USERS_CONFIG, u); + return config; + }) + .collect(Collectors.toList()); + } catch (IOException ex) { + throw new ConfigException("Cannot read users", ex); + } + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/config/LocalFitbitUser.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/config/LocalFitbitUser.java new file mode 100644 index 00000000..6e2cae7d --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/config/LocalFitbitUser.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.config; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.util.regex.Pattern; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.OAuth2UserCredentials; + +@JsonInclude(Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) +public class LocalFitbitUser implements FitbitUser { + private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[^a-zA-Z0-9_-]"); + private String id; + private String fitbitUserId; + private String projectId; + private String userId; + private String sourceId; + private Instant startDate = Instant.parse("2017-01-01T00:00:00Z"); + private Instant endDate = Instant.MAX; + + @JsonProperty("oauth2") + private OAuth2UserCredentials oauth2Credentials = new OAuth2UserCredentials(); + + @JsonIgnore + private SchemaAndValue observationKey; + + @Override + public String getId() { + return id; + } + + @JsonSetter + public void setId(String id) { + this.id = ILLEGAL_CHARACTERS_PATTERN.matcher(id).replaceAll("-"); + } + + public String getFitbitUserId() { + return fitbitUserId; + } + + public String getProjectId() { + return projectId; + } + + public String getUserId() { + return userId; + } + + public Instant getStartDate() { + return startDate; + } + + public Instant getEndDate() { + return endDate; + } + + public String getSourceId() { + return sourceId; + } + + public OAuth2UserCredentials getOAuth2Credentials() { + return this.oauth2Credentials; + } + + public void setOauth2Credentials(OAuth2UserCredentials oauth2Credentials) { + this.oauth2Credentials = oauth2Credentials; + } + + public LocalFitbitUser copy() { + LocalFitbitUser copy = new LocalFitbitUser(); + copy.id = id; + copy.fitbitUserId = fitbitUserId; + copy.projectId = projectId; + copy.userId = userId; + copy.startDate = startDate; + copy.endDate = endDate; + copy.sourceId = sourceId; + copy.oauth2Credentials = oauth2Credentials; + return copy; + } + + public synchronized SchemaAndValue getObservationKey(AvroData avroData) { + if (observationKey == null) { + observationKey = FitbitUser.computeObservationKey(avroData, this); + } + return observationKey; + } + + @Override + public String toString() { + return "LocalFitbitUser{" + "id='" + id + '\'' + + ", fitbitUserId='" + fitbitUserId + '\'' + + ", projectId='" + projectId + '\'' + + ", userId='" + userId + '\'' + + ", sourceId='" + sourceId + '\'' + + '}'; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java new file mode 100644 index 00000000..e3496985 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java @@ -0,0 +1,132 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.avro.generic.IndexedRecord; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.request.RestRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class to help convert Fitbit data to Avro Data. + */ +public abstract class FitbitAvroConverter implements PayloadToSourceRecordConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitAvroConverter.class); + private static final Map TIME_UNIT_MAP = new HashMap<>(); + + static { + TIME_UNIT_MAP.put("minute", TimeUnit.MINUTES); + TIME_UNIT_MAP.put("second", TimeUnit.SECONDS); + TIME_UNIT_MAP.put("hour", TimeUnit.HOURS); + TIME_UNIT_MAP.put("day", TimeUnit.DAYS); + TIME_UNIT_MAP.put("millisecond", TimeUnit.MILLISECONDS); + TIME_UNIT_MAP.put("nanosecond", TimeUnit.NANOSECONDS); + TIME_UNIT_MAP.put("microsecond", TimeUnit.MICROSECONDS); + } + + private final AvroData avroData; + + public FitbitAvroConverter(AvroData avroData) { + this.avroData = avroData; + } + + @Override + public Collection convert( + RestRequest restRequest, Response response) throws IOException { + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("Failed to read body"); + } + JsonNode activities = JSON_READER.readTree(body.charStream()); + + FitbitUser user = ((FitbitRestRequest) restRequest).getUser(); + final SchemaAndValue key = user.getObservationKey(avroData); + double timeReceived = System.currentTimeMillis() / 1000d; + + return processRecords((FitbitRestRequest)restRequest, activities, timeReceived) + .filter(Objects::nonNull) + .map(t -> { + SchemaAndValue avro = avroData.toConnectData(t.value.getSchema(), t.value); + Map offset = Collections.singletonMap( + TIMESTAMP_OFFSET_KEY, t.sourceOffset.toEpochMilli()); + + return new SourceRecord(restRequest.getPartition(), offset, t.topic, + key.schema(), key.value(), avro.schema(), avro.value()); + }) + .collect(Collectors.toList()); + } + + /** Process the JSON records generated by given request. */ + protected abstract Stream processRecords( + FitbitRestRequest request, + JsonNode root, + double timeReceived); + + /** Get Fitbit dataset interval used in some intraday API calls. */ + protected static int getRecordInterval(JsonNode root, int defaultValue) { + JsonNode type = root.get("datasetType"); + JsonNode interval = root.get("datasetInterval"); + if (type == null || interval == null) { + logger.warn("Failed to get data interval; using {} instead", defaultValue); + return defaultValue; + } + return (int)TIME_UNIT_MAP + .getOrDefault(type.asText(), TimeUnit.SECONDS) + .toSeconds(interval.asLong()); + } + + /** Converts an iterable (like a JsonNode containing an array) to a stream. */ + protected static Stream iterableToStream(Iterable iter) { + return StreamSupport.stream(iter.spliterator(), false); + } + + /** Single value for a topic. */ + protected static class TopicData { + Instant sourceOffset; + final String topic; + final IndexedRecord value; + + public TopicData(Instant sourceOffset, String topic, IndexedRecord value) { + this.sourceOffset = sourceOffset; + this.topic = topic; + this.value = value; + } + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java new file mode 100644 index 00000000..14d75d08 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradayHeartRate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitIntradayHeartRateAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger( + FitbitIntradayHeartRateAvroConverter.class); + private String heartRateTopic; + + public FitbitIntradayHeartRateAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + heartRateTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayHeartRateTopic(); + logger.info("Using heart rate topic {}", heartRateTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode intraday = root.get("activities-heart-intraday"); + if (intraday == null) { + return Stream.empty(); + } + + JsonNode dataset = intraday.get("dataset"); + if (dataset == null) { + return Stream.empty(); + } + + int interval = getRecordInterval(intraday, 1); + + // Used as the date to convert the local times in the dataset to absolute times. + ZonedDateTime startDate = request.getDateRange().start(); + + return iterableToStream(dataset) + .map(tryOrNull(activity -> { + Instant time = startDate.with(LocalTime.parse(activity.get("time").asText())) + .toInstant(); + + FitbitIntradayHeartRate heartRate = new FitbitIntradayHeartRate( + time.toEpochMilli() / 1000d, + timeReceived, + interval, + activity.get("value").asInt()); + + return new TopicData(time, heartRateTopic, heartRate); + }, (a, ex) -> logger.warn( + "Failed to convert heart rate from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), a, ex))); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java new file mode 100644 index 00000000..2381afa4 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradaySteps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitIntradayStepsAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger( + FitbitIntradayStepsAvroConverter.class); + + private String stepTopic; + + public FitbitIntradayStepsAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + stepTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayStepsTopic(); + logger.info("Using step topic {}", stepTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode intraday = root.get("activities-steps-intraday"); + if (intraday == null) { + return Stream.empty(); + } + + JsonNode dataset = intraday.get("dataset"); + if (dataset == null) { + return Stream.empty(); + } + + int interval = getRecordInterval(intraday, 60); + + // Used as the date to convert the local times in the dataset to absolute times. + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(dataset) + .map(tryOrNull(activity -> { + + Instant time = startDate + .with(LocalTime.parse(activity.get("time").asText())) + .toInstant(); + + FitbitIntradaySteps steps = new FitbitIntradaySteps( + time.toEpochMilli() / 1000d, + timeReceived, + interval, + activity.get("value").asInt()); + + return new TopicData(time, stepTopic, steps); + }, (a, ex) -> logger.warn( + "Failed to convert steps from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), a, ex))); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java new file mode 100644 index 00000000..48ebfa72 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java @@ -0,0 +1,139 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute.DATE_TIME_FORMAT; +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.avro.generic.IndexedRecord; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitSleepClassic; +import org.radarcns.connector.fitbit.FitbitSleepClassicLevel; +import org.radarcns.connector.fitbit.FitbitSleepStage; +import org.radarcns.connector.fitbit.FitbitSleepStageLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitSleepAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitSleepAvroConverter.class); + + private static final Map CLASSIC_MAP = new HashMap<>(); + private static final Map STAGES_MAP = new HashMap<>(); + static { + CLASSIC_MAP.put("awake", FitbitSleepClassicLevel.AWAKE); + CLASSIC_MAP.put("asleep", FitbitSleepClassicLevel.ASLEEP); + CLASSIC_MAP.put("restless", FitbitSleepClassicLevel.RESTLESS); + + STAGES_MAP.put("awake", FitbitSleepStageLevel.AWAKE); + STAGES_MAP.put("rem", FitbitSleepStageLevel.REM); + STAGES_MAP.put("deep", FitbitSleepStageLevel.DEEP); + STAGES_MAP.put("light", FitbitSleepStageLevel.LIGHT); + } + + private String sleepStagesTopic; + private String sleepClassicTopic; + + public FitbitSleepAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + sleepStagesTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepStagesTopic(); + sleepClassicTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepClassicTopic(); + + logger.info("Using sleep topic {} and {}", sleepStagesTopic, sleepClassicTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode meta = root.get("meta"); + if (meta != null) { + JsonNode state = meta.get("state"); + if (state != null && meta.get("state").asText().equals("pending")) { + return Stream.empty(); + } + } + JsonNode sleepArray = root.get("sleep"); + if (sleepArray == null) { + return Stream.empty(); + } + + return iterableToStream(sleepArray) + .sorted(Comparator.comparing(s -> s.get("startTime").asText())) + .flatMap(tryOrNull(s -> { + Instant startTime = Instant.from(DATE_TIME_FORMAT.parse(s.get("startTime").asText())); + boolean isStages = s.get("type") == null || s.get("type").asText().equals("stages"); + + // use an intermediate offset for all records but the last. Since the query time + // depends only on the start time of a sleep stages group, this will reprocess the entire + // sleep stages group if something goes wrong while processing. + Instant intermediateOffset = startTime.minus(Duration.ofSeconds(1)); + + List allRecords = iterableToStream(s.get("levels").get("data")) + .map(d -> { + IndexedRecord sleep; + String topic; + + String dateTime = d.get("dateTime").asText(); + int duration = d.get("seconds").asInt(); + String level = d.get("level").asText(); + + if (isStages) { + sleep = new FitbitSleepStage( + dateTime, + timeReceived, + duration, + STAGES_MAP.getOrDefault(level, FitbitSleepStageLevel.UNKNOWN)); + topic = sleepStagesTopic; + } else { + sleep = new FitbitSleepClassic( + dateTime, + timeReceived, + duration, + CLASSIC_MAP.getOrDefault(level, FitbitSleepClassicLevel.UNKNOWN)); + topic = sleepClassicTopic; + } + + return new TopicData(intermediateOffset, topic, sleep); + }) + .collect(Collectors.toList()); + + // The final group gets the actual offset, to ensure that the group does not get queried + // again. + allRecords.get(allRecords.size() - 1).sourceOffset = startTime; + + return allRecords.stream(); + }, (s, ex) -> logger.warn( + "Failed to convert sleep patterns from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), s, ex))); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java new file mode 100644 index 00000000..01c77d8f --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitTimeZoneAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitTimeZoneAvroConverter.class); + + private String timeZoneTopic; + + public FitbitTimeZoneAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + timeZoneTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitTimeZoneTopic(); + logger.info("Using timezone topic {}", timeZoneTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, + JsonNode root, + double timeReceived) { + JsonNode user = root.get("user"); + if (user == null) { + logger.warn("Failed to get timezone from {}, {}", request.getRequest().url(), root); + return Stream.empty(); + } + JsonNode offsetNode = user.get("offsetFromUTCMillis"); + Integer offset = offsetNode == null ? null : (int) (offsetNode.asLong() / 1000L); + + FitbitTimeZone timeZone = new FitbitTimeZone(timeReceived, offset); + + return Stream.of(new TopicData(request.getDateRange().start().toInstant(), + timeZoneTopic, timeZone)); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java new file mode 100644 index 00000000..08398fd9 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java @@ -0,0 +1,107 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.request; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import io.confluent.connect.avro.AvroData; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import okhttp3.OkHttpClient; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.route.FitbitIntradayHeartRateRoute; +import org.radarbase.connect.rest.fitbit.route.FitbitIntradayStepsRoute; +import org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute; +import org.radarbase.connect.rest.fitbit.route.FitbitTimeZoneRoute; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.radarbase.connect.rest.request.RequestGeneratorRouter; +import org.radarbase.connect.rest.request.RequestRoute; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Generate all requests for Fitbit API. + */ +public class FitbitRequestGenerator extends RequestGeneratorRouter { + public static final JsonFactory JSON_FACTORY = new JsonFactory(); + public static final ObjectReader JSON_READER = new ObjectMapper(JSON_FACTORY).reader(); + private static final Logger logger = LoggerFactory.getLogger(FitbitRequestGenerator.class); + + private OkHttpClient baseClient; + private final Map clients; + private FitbitUserRepository userRepository; + private List routes; + + public FitbitRequestGenerator() { + clients = new HashMap<>(); + } + + @Override + public Stream routes() { + return this.routes.stream(); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + FitbitRestSourceConnectorConfig config1 = (FitbitRestSourceConnectorConfig) config; + this.baseClient = new OkHttpClient(); + + AvroData avroData = new AvroData(20); + this.userRepository = config1.getFitbitUserRepository(); + this.routes = Arrays.asList( + new FitbitIntradayStepsRoute(this, userRepository, avroData), + new FitbitSleepRoute(this, userRepository, avroData), + new FitbitIntradayHeartRateRoute(this, userRepository, avroData), + new FitbitTimeZoneRoute(this, userRepository, avroData) + ); + + super.initialize(config); + } + + public OkHttpClient getClient(FitbitUser user) { + return clients.computeIfAbsent(user.getId(), u -> baseClient.newBuilder() + .authenticator(new TokenAuthenticator(user, userRepository)) + .build()); + } + + public Map> getPartitions(String route) { + try { + return userRepository.stream() + .collect(Collectors.toMap(FitbitUser::getId, u -> getPartition(route, u))); + } catch (IOException e) { + logger.warn("Failed to initialize user partitions for route {}: {}", route, e.toString()); + return Collections.emptyMap(); + } + } + + public Map getPartition(String route, FitbitUser user) { + Map partition = new HashMap<>(4); + partition.put("user", user.getId()); + partition.put("route", route); + return partition; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java new file mode 100644 index 00000000..2ede8a84 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.request; + +import java.util.Map; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.util.DateRange; +import org.radarbase.connect.rest.request.RequestRoute; +import org.radarbase.connect.rest.request.RestRequest; + +/** + * REST request taking into account the user and offsets queried. The offsets are useful for + * defining what dates to poll (again). + */ +public class FitbitRestRequest extends RestRequest { + private final FitbitUser user; + private final DateRange dateRange; + + public FitbitRestRequest( + RequestRoute requestRoute, Request request, FitbitUser user, + Map partition, OkHttpClient client, DateRange dateRange) { + super(requestRoute, client, request, partition); + this.user = user; + this.dateRange = dateRange; + } + + public FitbitUser getUser() { + return user; + } + + public DateRange getDateRange() { + return dateRange; + } + + @Override + public String toString() { + return "FitbitRestRequest{" + + "url=" + getRequest().url() + + ", user=" + user + + ", dateRange=" + dateRange + + '}'; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/TokenAuthenticator.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/TokenAuthenticator.java new file mode 100644 index 00000000..6f9416a9 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/TokenAuthenticator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.request; + +import java.io.IOException; +import javax.ws.rs.NotAuthorizedException; +import okhttp3.Authenticator; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.Route; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Authenticator for Fitbit, which tries to refresh the access token if a request is unauthorized. + */ +public class TokenAuthenticator implements Authenticator { + private static final Logger logger = LoggerFactory.getLogger(TokenAuthenticator.class); + + private final FitbitUser user; + private final FitbitUserRepository userRepository; + + TokenAuthenticator(FitbitUser user, FitbitUserRepository userRepository) { + this.user = user; + this.userRepository = userRepository; + } + + @Override + public Request authenticate(Route requestRoute, Response response) throws IOException { + if (response.code() != 401) { + return null; + } + + try { + String newAccessToken = userRepository.refreshAccessToken(user); + + return response.request().newBuilder() + .header("Authorization", "Bearer " + newAccessToken) + .build(); + } catch (NotAuthorizedException ex) { + logger.error("Cannot get a new refresh token for user {}. Cancelling request.", user); + return null; + } + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java new file mode 100644 index 00000000..1ba3181b --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; +import static java.time.temporal.ChronoUnit.SECONDS; + +import io.confluent.connect.avro.AvroData; +import java.util.stream.Stream; +import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayHeartRateAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; + +public class FitbitIntradayHeartRateRoute extends FitbitPollingRoute { + private static final String ROUTE_NAME = "heart_rate"; + private final FitbitIntradayHeartRateAvroConverter converter; + + public FitbitIntradayHeartRateRoute(FitbitRequestGenerator generator, + FitbitUserRepository userRepository, AvroData avroData) { + super(generator, userRepository, ROUTE_NAME); + this.converter = new FitbitIntradayHeartRateAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/activities/heart/date/%s/1d/1sec/time/%s/%s.json?timezone=UTC"; + } + + protected Stream createRequests(FitbitUser user) { + return startDateGenerator(getOffset(user).plus(ONE_SECOND).truncatedTo(SECONDS)) + .map(dateRange -> newRequest(user, dateRange, + user.getFitbitUserId(), DATE_FORMAT.format(dateRange.start()), + ISO_LOCAL_TIME.format(dateRange.start()), + ISO_LOCAL_TIME.format(dateRange.end().truncatedTo(SECONDS)))); + } + + @Override + public FitbitIntradayHeartRateAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java new file mode 100644 index 00000000..3ea7579d --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static java.time.temporal.ChronoUnit.MINUTES; + +import io.confluent.connect.avro.AvroData; +import java.time.temporal.TemporalAmount; +import java.util.stream.Stream; +import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayStepsAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; + +public class FitbitIntradayStepsRoute extends FitbitPollingRoute { + private static final TemporalAmount ONE_MINUTE = MINUTES.getDuration(); + + private final FitbitIntradayStepsAvroConverter converter; + + public FitbitIntradayStepsRoute(FitbitRequestGenerator generator, + FitbitUserRepository userRepository, AvroData avroData) { + super(generator, userRepository, "intraday_steps"); + this.converter = new FitbitIntradayStepsAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/activities/steps/date/%s/1d/1min/time/%s/%s.json?timezone=UTC"; + } + + protected Stream createRequests(FitbitUser user) { + return startDateGenerator(this.getOffset(user).plus(ONE_MINUTE).truncatedTo(MINUTES)) + .map(dateRange -> newRequest(user, dateRange, + user.getFitbitUserId(), DATE_FORMAT.format(dateRange.start()), + TIME_FORMAT.format(dateRange.start()), TIME_FORMAT.format(dateRange.end()))); + } + + @Override + public FitbitIntradayStepsAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java new file mode 100644 index 00000000..387a7462 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java @@ -0,0 +1,329 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.DAYS; +import static java.time.temporal.ChronoUnit.NANOS; +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.TIMESTAMP_OFFSET_KEY; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAmount; +import java.util.AbstractMap; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.ws.rs.NotAuthorizedException; +import okhttp3.Request; +import okhttp3.Response; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; +import org.radarbase.connect.rest.request.PollingRequestRoute; +import org.radarbase.connect.rest.request.RestRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Route for regular polling. + * + *

The algorithm uses the following polling times: + * 1. do not try polling until getLastPoll() + getPollInterval() + * 2. if that has passed, determine for each user when to poll again. Per user: + * 1. if a successful call was made that returned data, take the last successful offset and after + * getLookbackTime() has passed, poll again. + * 2. if a successful call was made that did not return data, take the last query interval + * and start cycling up from the last successful record, starting no further than + * HISTORICAL_TIME + * + *

Conditions that should be met: + * 1. Do not poll more frequently than once every getPollInterval(). + * 2. On first addition of a user, poll its entire history + * 3. If the history of a user has been scanned, do not look back further than + * {@code HISTORICAL_TIME}. This ensures fewer operations under normal operations, where Fitbit + * data is fairly frequently updated. + * 4. If there was data for a certain date time in an API, earlier date times are not polled. This + * prevents duplicate data. + * 5. From after the latest known date time, the history of the user is regularly inspected for new + * records. + * 6, All of the recent history is simultaneously inspected to prevent reading only later data in + * a single batch that is added to the API. + * 6. Do not try to read any records for the last {@code getLookbackTime()}. This lookback time ensures that + * when new records are added by another device within the {@code getLookbackTime()} period, they are also + * added. + * 7. When a too many records exception occurs, do not poll for given user for + * {@code TOO_MANY_REQUESTS_COOLDOWN}. + */ +public abstract class FitbitPollingRoute implements PollingRequestRoute { + protected static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE; + protected static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("HH:mm"); + protected static final Duration LOOKBACK_TIME = Duration.ofDays(1); // 1 day + protected static final long HISTORICAL_TIME_DAYS = 14L; + protected static final Duration TOO_MANY_REQUESTS_COOLDOWN = Duration.ofHours(1); + protected static final Duration ONE_DAY = DAYS.getDuration(); + protected static final Duration ONE_NANO = NANOS.getDuration(); + protected static final TemporalAmount ONE_SECOND = SECONDS.getDuration(); + + private static final Logger logger = LoggerFactory.getLogger(FitbitSleepRoute.class); + + /** Committed offsets. */ + private Map offsets; + + private final Map> partitions; + private final Map lastPollPerUser; + private final FitbitRequestGenerator generator; + private final FitbitUserRepository userRepository; + private final String routeName; + private Duration pollInterval; + private Instant lastPoll; + private String baseUrl; + private long maxUsersPerPoll; + private Duration pollIntervalPerUser; + + public FitbitPollingRoute( + FitbitRequestGenerator generator, + FitbitUserRepository userRepository, + String routeName) { + this.generator = generator; + this.userRepository = userRepository; + this.offsets = new HashMap<>(); + this.partitions = new HashMap<>(generator.getPartitions(routeName)); + this.routeName = routeName; + this.lastPoll = Instant.MIN; + this.lastPollPerUser = new HashMap<>(); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + this.pollInterval = config.getPollInterval(); + this.baseUrl = config.getUrl(); + this.maxUsersPerPoll = config.getMaxUsersPerPoll(); + this.pollIntervalPerUser = config.getPollIntervalPerUser(); + this.converter().initialize(config); + } + + @Override + public void requestSucceeded(RestRequest request, SourceRecord record) { + String userKey = ((FitbitRestRequest) request).getUser().getId(); + Instant offset = Instant.ofEpochMilli((Long) record.sourceOffset().get(TIMESTAMP_OFFSET_KEY)); + offsets.put(userKey, offset); + } + + @Override + public void requestEmpty(RestRequest request) { + FitbitRestRequest fitbitRequest = (FitbitRestRequest) request; + Instant endOffset = fitbitRequest.getDateRange().end().toInstant(); + if (DAYS.between(endOffset, lastPoll) >= HISTORICAL_TIME_DAYS) { + String key = fitbitRequest.getUser().getId(); + offsets.put(key, endOffset); + } + } + + @Override + public void requestFailed(RestRequest request, Response response) { + if (response != null && response.code() == 429) { + FitbitUser user = ((FitbitRestRequest)request).getUser(); + logger.info("Too many requests for user {}. Backing off for {}", + user, TOO_MANY_REQUESTS_COOLDOWN); + lastPollPerUser.compute(user.getId(), (u, p) -> p == null + ? lastPoll.plus(TOO_MANY_REQUESTS_COOLDOWN) + : PollingRequestRoute.max( + p.plus(getPollIntervalPerUser()), lastPoll.plus(TOO_MANY_REQUESTS_COOLDOWN))); + } else { + logger.warn("Failed to make request {}", request); + } + } + + /** + * Actually construct requests, based on the current offset + * @param user Fitbit user + * @return request to make + */ + protected abstract Stream createRequests(FitbitUser user); + + @Override + public Stream requests() { + lastPoll = Instant.now(); + try { + return userRepository.stream() + .map(u -> new AbstractMap.SimpleImmutableEntry<>(u, nextPoll(u))) + .filter(u -> lastPoll.isAfter(u.getValue())) + .sorted(Comparator.comparing(Map.Entry::getValue)) + .limit(maxUsersPerPoll) + .flatMap(u -> { + lastPollPerUser.put(u.getKey().getId(), lastPoll); + return this.createRequests(u.getKey()); + }) + .filter(Objects::nonNull); + } catch (IOException e) { + logger.warn("Cannot read users"); + return Stream.empty(); + } + } + + private Map getPartition(FitbitUser user) { + return partitions.computeIfAbsent(user.getId(), + k -> generator.getPartition(routeName, user)); + } + + /** + * Create a FitbitRestRequest for given arguments. + * @param user Fitbit user + * @param dateRange dates that may be queried in the request + * @param urlFormatArgs format arguments to {@link #getUrlFormat(String)}. + * @return request or {@code null} if the authorization cannot be arranged. + */ + protected FitbitRestRequest newRequest(FitbitUser user, DateRange dateRange, + Object... urlFormatArgs) { + Request.Builder builder = new Request.Builder() + .url(String.format(getUrlFormat(baseUrl), urlFormatArgs)); + try { + Request request = builder + .header("Authorization", "Bearer " + userRepository.getAccessToken(user)) + .build(); + return new FitbitRestRequest(this, request, user, getPartition(user), + generator.getClient(user), dateRange); + } catch (NotAuthorizedException | IOException ex) { + logger.warn("User {} does not have a configured access token: {}. Skipping.", + user, ex.toString()); + return null; + } + } + + @Override + public void setOffsetStorageReader(OffsetStorageReader offsetStorageReader) { + if (offsetStorageReader != null) { + offsets = offsetStorageReader.offsets(partitions.values()).entrySet().stream() + .filter(e -> e.getValue() != null && e.getValue().containsKey(TIMESTAMP_OFFSET_KEY)) + .collect(Collectors.toMap( + e -> (String) e.getKey().get("user"), + e -> Instant.ofEpochMilli((Long) e.getValue().get(TIMESTAMP_OFFSET_KEY)))); + } else { + logger.warn("Offset storage reader is null, will resume from an empty state."); + } + } + + @Override + public Duration getPollInterval() { + return pollInterval; + } + + @Override + public Stream nextPolls() { + try { + return userRepository.stream() + .map(this::nextPoll); + } catch (IOException e) { + logger.warn("Failed to read users for polling interval: {}", e.toString()); + return Stream.of(getLastPoll().plus(getPollInterval())); + } + } + + public Instant getLastPoll() { + return lastPoll; + } + + protected Instant getOffset(FitbitUser user) { + return offsets.getOrDefault(user.getId(), user.getStartDate().minus(ONE_NANO)); + } + + /** + * URL String format. The format arguments should be provided to + * {@link #newRequest(FitbitUser, Instant, Instant, Object...)} + */ + protected abstract String getUrlFormat(String baseUrl); + + /** + * Get the poll interval for a single user on a single route. + */ + protected Duration getPollIntervalPerUser() { + return pollIntervalPerUser; + } + + /** + * Time that should not be polled to avoid duplicate data. + */ + protected Duration getLookbackTime() { + return LOOKBACK_TIME; + } + + /** + * Next time that given user should be polled. + */ + protected Instant nextPoll(FitbitUser user) { + Instant offset = getOffset(user); + if (offset.isAfter(user.getEndDate())) { + return Instant.MAX; + } else { + Instant nextPoll = lastPollPerUser.getOrDefault(user.getId(), Instant.MIN) + .plus(getPollIntervalPerUser()); + return PollingRequestRoute.max(offset.plus(getLookbackTime()), nextPoll); + } + } + + /** + * Generate one date per day, using UTC time zone. The first date will have the time from the + * given startDate. Following time stamps will start at 00:00. This will not up to the date of + * {@code getLookbackTime()} (exclusive). + */ + Stream startDateGenerator(Instant startDate) { + Instant lookBack = lastPoll.minus(getLookbackTime()); + + ZonedDateTime dateTime = startDate.atZone(UTC); + ZonedDateTime lookBackDate = lookBack.atZone(UTC); + ZonedDateTime lookBackDateStart = lookBackDate.truncatedTo(DAYS); + + // last date to poll is equal to the last polled date + if (lookBackDateStart.equals(dateTime.truncatedTo(DAYS))) { + if (lookBack.isAfter(startDate)) { + return Stream.of(new DateRange(dateTime, lookBackDate)); + } else { + return Stream.empty(); + } + } else { + long numElements = DAYS.between(startDate, lookBack); + + Stream elements = Stream + .iterate(dateTime, t -> t.plus(ONE_DAY).truncatedTo(DAYS)) + .limit(numElements) + .map(s -> new DateRange(s, s.plus(ONE_DAY).truncatedTo(DAYS).minus(ONE_NANO))); + + // we're polling at exactly 00:00, should not poll the last date + if (lookBackDateStart.equals(lookBackDate)) { + return elements; + } else { + // we're polling startDate - night, x days, lastDay - lookBackDate + return Stream.concat(elements, + Stream.of(new DateRange(lookBackDateStart, lookBackDate))); + } + } + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java new file mode 100644 index 00000000..532ab5ff --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.SECONDS; + +import io.confluent.connect.avro.AvroData; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.stream.Stream; +import org.radarbase.connect.rest.fitbit.converter.FitbitSleepAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; + +public class FitbitSleepRoute extends FitbitPollingRoute { + public static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE_TIME + .withZone(UTC); + private static final Duration SLEEP_POLL_INTERVAL = Duration.ofDays(1); + + private final FitbitSleepAvroConverter converter; + + public FitbitSleepRoute(FitbitRequestGenerator generator, FitbitUserRepository userRepository, + AvroData avroData) { + super(generator, userRepository, "sleep"); + converter = new FitbitSleepAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1.2/user/%s/sleep/list.json?sort=asc&afterDate=%s&limit=100&offset=0"; + } + + /** + * Actually construct a request, based on the current offset + * @param user Fitbit user + * @return request to make + */ + protected Stream createRequests(FitbitUser user) { + ZonedDateTime startDate = this.getOffset(user).plus(ONE_SECOND) + .atZone(UTC) + .truncatedTo(SECONDS); + + return Stream.of(newRequest(user, new DateRange(startDate, ZonedDateTime.now(UTC)), + user.getFitbitUserId(), DATE_TIME_FORMAT.format(startDate))); + } + + + @Override + protected Duration getPollIntervalPerUser() { + return SLEEP_POLL_INTERVAL; + } + + @Override + public FitbitSleepAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java new file mode 100644 index 00000000..9b1a52ac --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static java.time.ZoneOffset.UTC; + +import io.confluent.connect.avro.AvroData; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.fitbit.converter.FitbitTimeZoneAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.FitbitUser; +import org.radarbase.connect.rest.fitbit.user.FitbitUserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; + +public class FitbitTimeZoneRoute extends FitbitPollingRoute { + protected static final Duration TIME_ZONE_POLL_INTERVAL = Duration.ofHours(1); + + private final FitbitTimeZoneAvroConverter converter; + + public FitbitTimeZoneRoute(FitbitRequestGenerator generator, + FitbitUserRepository userRepository, AvroData avroData) { + super(generator, userRepository, "timezone"); + this.converter = new FitbitTimeZoneAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/profile.json"; + } + + protected Stream createRequests(FitbitUser user) { + ZonedDateTime now = ZonedDateTime.now(UTC); + return Stream.of(newRequest(user, new DateRange(now, now), user.getFitbitUserId())); + } + + @Override + public FitbitTimeZoneAvroConverter converter() { + return converter; + } + + @Override + protected Duration getPollIntervalPerUser() { + return TIME_ZONE_POLL_INTERVAL; + } + + @Override + protected Duration getLookbackTime() { + return Duration.ZERO; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUser.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUser.java new file mode 100644 index 00000000..b0feca23 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUser.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.user; + +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.radarcns.kafka.ObservationKey; + +public interface FitbitUser { + String getId(); + String getFitbitUserId(); + String getProjectId(); + String getUserId(); + Instant getStartDate(); + Instant getEndDate(); + String getSourceId(); + SchemaAndValue getObservationKey(AvroData avroData); + + static SchemaAndValue computeObservationKey(AvroData avroData, FitbitUser user) { + return avroData.toConnectData( + ObservationKey.getClassSchema(), + new ObservationKey(user.getProjectId(), user.getUserId(), user.getSourceId())); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUserRepository.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUserRepository.java new file mode 100644 index 00000000..02c50b92 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/FitbitUserRepository.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.user; + +import java.io.IOException; +import java.util.stream.Stream; +import javax.ws.rs.NotAuthorizedException; +import org.radarbase.connect.rest.config.RestSourceTool; + +/** + * User repository for Fitbit users. + */ +public interface FitbitUserRepository extends RestSourceTool { + /** + * Get specified Fitbit user. + * @throws IOException if the user cannot be retrieved from the repository. + */ + FitbitUser get(String key) throws IOException; + + /** + * Get all relevant Fitbit users. + * @throws IOException if the list cannot be retrieved from the repository. + */ + Stream stream() throws IOException; + + /** + * Get the current access token of given user. If it has expired, a new access token will + * be requested. If the server indicates the token is invalid, call + * {@link #refreshAccessToken(FitbitUser)} instead. + * + * @throws IOException if the new access token cannot be retrieved from the repository. + * @throws NotAuthorizedException if the refresh token is no longer valid. Manual action + * should be taken to get a new refresh token. + * @throws java.util.NoSuchElementException if the user does not exists in this repository. + */ + String getAccessToken(FitbitUser user) throws IOException, NotAuthorizedException; + + /** + * Refresh the access token of given user. + * @throws IOException if the new access token cannot be retrieved from the repository. + * @throws NotAuthorizedException if the refresh token is no longer valid. Manual action + * should be taken to get a new refresh token. + * @throws java.util.NoSuchElementException if the user does not exists in this repository. + */ + String refreshAccessToken(FitbitUser user) throws IOException, NotAuthorizedException; +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/OAuth2UserCredentials.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/OAuth2UserCredentials.java new file mode 100644 index 00000000..a27b8d6c --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/OAuth2UserCredentials.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.user; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.Duration; +import java.time.Instant; + +public class OAuth2UserCredentials { + private static final Duration DEFAULT_EXPIRY = Duration.ofHours(1); + private static final Duration EXPIRY_TIME_MARGIN = Duration.ofMinutes(5); + + @JsonProperty + private final String accessToken; + @JsonProperty + private final String refreshToken; + @JsonProperty + private final Instant expiresAt; + + public OAuth2UserCredentials() { + this(null, null, (Instant)null); + } + + @JsonCreator + public OAuth2UserCredentials( + @JsonProperty("refreshToken") String refreshToken, + @JsonProperty("accessToken") String accessToken, + @JsonProperty("expiresAt") Instant expiresAt) { + this.refreshToken = refreshToken; + this.accessToken = accessToken; + this.expiresAt = expiresAt == null ? getExpiresAt(DEFAULT_EXPIRY) : expiresAt; + } + + public OAuth2UserCredentials(String refreshToken, String accessToken, Long expiresIn) { + this(refreshToken, accessToken, getExpiresAt( + expiresIn != null && expiresIn > 0L ? Duration.ofSeconds(expiresIn) : DEFAULT_EXPIRY)); + } + + public String getAccessToken() { + return accessToken; + } + + public boolean hasRefreshToken() { + return refreshToken != null && !refreshToken.isEmpty(); + } + + public String getRefreshToken() { + return refreshToken; + } + + protected static Instant getExpiresAt(Duration expiresIn) { + return Instant.now() + .plus(expiresIn) + .minus(EXPIRY_TIME_MARGIN); + } + + @JsonIgnore + public boolean isAccessTokenExpired() { + return Instant.now().isAfter(expiresAt); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/YamlFitbitUserRepository.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/YamlFitbitUserRepository.java new file mode 100644 index 00000000..108851c6 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/YamlFitbitUserRepository.java @@ -0,0 +1,343 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.user; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrRethrow; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.ws.rs.NotAuthorizedException; +import okhttp3.FormBody; +import okhttp3.Headers; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.kafka.common.config.ConfigException; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.config.LocalFitbitUser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * User repository that reads and writes configuration of YAML files in a local directory. The + * directory will be recursively scanned for all YAML files. Those should all contain + * {@link LocalFitbitUser} serializations. + */ +public class YamlFitbitUserRepository implements FitbitUserRepository { + private static final Logger logger = LoggerFactory.getLogger(YamlFitbitUserRepository.class); + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(YAML_FACTORY); + static { + YAML_MAPPER.registerModule(new JavaTimeModule()); + YAML_MAPPER.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } + private static final ObjectReader USER_READER = YAML_MAPPER.readerFor(LocalFitbitUser.class); + private static final ObjectWriter USER_WRITER = YAML_MAPPER.writerFor(LocalFitbitUser.class); + private static final Duration FETCH_THRESHOLD = Duration.ofHours(1L); + + private static final int NUM_RETRIES = 10; + + + private final OkHttpClient client; + + private Set configuredUsers; + private Headers headers; + private ConcurrentMap users = new ConcurrentHashMap<>(); + private final AtomicReference nextFetch = new AtomicReference<>(Instant.MIN); + private Path credentialsDir; + + public YamlFitbitUserRepository() { + this.client = new OkHttpClient(); + } + + @Override + public FitbitUser get(String key) { + updateUsers(); + LockedUser user = users.get(key); + if (user == null) { + return null; + } else { + return user.apply(LocalFitbitUser::copy); + } + } + + private void updateUsers() { + Instant nextFetchTime = nextFetch.get(); + Instant now = Instant.now(); + if (!now.isAfter(nextFetchTime) + || !nextFetch.compareAndSet(nextFetchTime, now.plus(FETCH_THRESHOLD))) { + return; + } + + try { + Map newMap = Files.walk(credentialsDir) + .filter(p -> Files.isRegularFile(p) + && p.getFileName().toString().toLowerCase().endsWith(".yml")) + .map(tryOrRethrow(p -> new LockedUser(USER_READER.readValue(p.toFile()), p))) + .collect(Collectors.toMap(l -> l.user.getId(), Function.identity())); + + users.keySet().removeIf(u -> !newMap.containsKey(u)); + newMap.forEach(users::putIfAbsent); + } catch (IOException | RuntimeException ex) { + logger.error("Failed to read user directories: {}", ex.toString()); + } + } + + @Override + public Stream stream() { + updateUsers(); + + Stream users = this.users.values().stream() + .filter(lockedTest(u -> u.getOAuth2Credentials().hasRefreshToken())); + if (!configuredUsers.isEmpty()) { + users = users.filter(lockedTest(u -> configuredUsers.contains(u.getId()))); + } + return users.map(lockedApply(LocalFitbitUser::copy)); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + this.credentialsDir = ((FitbitRestSourceConnectorConfig) config) + .getFitbitUserCredentialsPath(); + try { + this.credentialsDir = ((FitbitRestSourceConnectorConfig)config).getFitbitUserCredentialsPath(); + Files.createDirectories(this.credentialsDir); + } catch (IOException ex) { + throw new ConfigException("Failed to read user directory " + credentialsDir, ex); + } + FitbitRestSourceConnectorConfig fitbitConfig = (FitbitRestSourceConnectorConfig) config; + configuredUsers = new HashSet<>(fitbitConfig.getFitbitUsers()); + headers = ((FitbitRestSourceConnectorConfig) config).getClientCredentials(); + } + + @Override + public String getAccessToken(FitbitUser user) throws IOException, NotAuthorizedException { + updateUsers(); + LockedUser actualUser = this.users.get(user.getId()); + if (actualUser == null) { + throw new NoSuchElementException("User " + user + " is not present in this user repository."); + } + String currentToken = actualUser.apply(u -> { + if (!u.getOAuth2Credentials().isAccessTokenExpired()) { + return u.getOAuth2Credentials().getAccessToken(); + } else { + return null; + } + }); + + return currentToken != null ? currentToken : refreshAccessToken(user); + } + + @Override + public String refreshAccessToken(FitbitUser user) throws IOException { + return refreshAccessToken(user, NUM_RETRIES); + } + + /** + * Refreshes the Fitbit access token on the current host, using the locally stored refresh token. + * If successful, the tokens are locally stored. + * If the refresh token is expired or invalid, the access token and the refresh token are set to + * null. + * @param user user to request access token for. + * @param retry number of retries before exiting. + * @return access token + * @throws IOException if the refresh fails + * @throws NotAuthorizedException if no refresh token is stored with the user or if the + * current refresh token is no longer valid. + */ + public synchronized String refreshAccessToken(FitbitUser user, int retry) throws IOException { + LockedUser actualUser = this.users.get(user.getId()); + if (actualUser == null) { + throw new NoSuchElementException("User " + user + " is not present in this user repository."); + } + String refreshToken = actualUser.apply(u -> u.getOAuth2Credentials().getRefreshToken()); + if (refreshToken == null || refreshToken.isEmpty()) { + throw new NotAuthorizedException("Refresh token is not set"); + } + Request request = new Request.Builder() + .url("https://api.fitbit.com/oauth2/token") + .headers(headers) + .post(new FormBody.Builder() + .add("grant_type", "refresh_token") + .add("refresh_token", refreshToken) + .build()) + .build(); + + try (Response response = client.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + + if (response.isSuccessful() && responseBody != null) { + JsonNode node; + try { + node = JSON_READER.readTree(responseBody.charStream()); + } catch (IOException ex) { + if (retry > 0) { + logger.warn("Failed to read OAuth 2.0 response: {}", ex.toString()); + return refreshAccessToken(user, retry - 1); + } + throw ex; + } + + JsonNode expiresInNode = node.get("expires_in"); + Long expiresIn = expiresInNode != null + ? expiresInNode.asLong() + : null; + + JsonNode accessTokenNode = node.get("access_token"); + JsonNode refreshTokenNode = node.get("refresh_token"); + if (accessTokenNode == null || refreshTokenNode == null) { + if (retry > 0) { + logger.warn("Failed to get access token in successful OAuth 2.0 request:" + + " access token or refresh token are missing"); + return refreshAccessToken(user, retry - 1); + } else { + throw new NotAuthorizedException("Did not get an access token"); + } + } + + actualUser.accept((u, p) -> { + if (!refreshToken.equals(u.getOAuth2Credentials().getRefreshToken())) { + // it was updated already by another thread. + return; + } + u.setOauth2Credentials(new OAuth2UserCredentials( + refreshTokenNode.asText(), accessTokenNode.asText(), expiresIn)); + store(p, u); + }); + } else if (response.code() == 400 || response.code() == 401) { + actualUser.accept((u, p) -> { + if (!refreshToken.equals(u.getOAuth2Credentials().getRefreshToken())) { + // it was updated already by another thread. + return; + } + u.setOauth2Credentials(new OAuth2UserCredentials()); + store(p, u); + }); + throw new NotAuthorizedException("Refresh token is no longer valid."); + } else { + String message = "Failed to request refresh token, with response HTTP status code " + + response.code(); + if (responseBody != null) { + message += " and content " + responseBody.string(); + } + throw new IOException(message); + } + } + return actualUser.apply(u -> u.getOAuth2Credentials().getAccessToken()); + } + + /** + * Store a user to given path. + * @param path path to store at. + * @param user use to store. + */ + private void store(Path path, LocalFitbitUser user) { + try { + Path temp = Files.createTempFile(user.getId(), ".tmp"); + try { + try (OutputStream out = Files.newOutputStream(temp)) { + synchronized (this) { + USER_WRITER.writeValue(out, user); + } + } + Files.move(temp, path, REPLACE_EXISTING); + } finally { + Files.deleteIfExists(temp); + } + } catch (IOException ex) { + logger.error("Failed to store user file: {}", ex.toString()); + } + } + + /** + * Local user that is protected by a multi-threading lock to avoid simultaneous IO + * and modifications. + */ + private final class LockedUser { + final Lock lock = new ReentrantLock(); + final LocalFitbitUser user; + final Path path; + + LockedUser(LocalFitbitUser user, Path path) { + this.user = user; + this.path = path; + } + + V apply(Function func) { + lock.lock(); + try { + return func.apply(user); + } finally { + lock.unlock(); + } + } + + void accept(BiConsumer consumer) { + lock.lock(); + try { + consumer.accept(user, path); + } finally { + lock.unlock(); + } + } + } + + private static Function lockedApply(Function func) { + return l -> l.apply(func); + } + + private static Predicate lockedTest(Predicate func) { + return l -> { + l.lock.lock(); + try { + return func.test(l.user); + } finally { + l.lock.unlock(); + } + }; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java new file mode 100644 index 00000000..b799f1bc --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java @@ -0,0 +1,71 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.util; + +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.Objects; + +public class DateRange { + private final ZonedDateTime start; + private final ZonedDateTime end; + + public DateRange(ZonedDateTime start, ZonedDateTime end) { + this.start = start; + this.end = end; + } + + public DateRange(ZonedDateTime start, TemporalAmount duration) { + this.start = start; + this.end = start.plus(duration); + } + + public ZonedDateTime start() { + return start; + } + + public ZonedDateTime end() { + return end; + } + + @Override + public String toString() { + return "DateRange{" + + "start=" + start + + ", end=" + end + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DateRange dateRange = (DateRange) o; + return Objects.equals(start, dateRange.start) && + Objects.equals(end, dateRange.end); + } + + @Override + public int hashCode() { + return Objects.hash(start, end); + } +} diff --git a/kafka-connect-fitbit-source/src/test/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRouteTest.java b/kafka-connect-fitbit-source/src/test/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRouteTest.java new file mode 100644 index 00000000..83a27842 --- /dev/null +++ b/kafka-connect-fitbit-source/src/test/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRouteTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.radarbase.connect.rest.fitbit.route.FitbitPollingRoute.DATE_FORMAT; +import static org.radarbase.connect.rest.fitbit.route.FitbitPollingRoute.TIME_FORMAT; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Test; + +class FitbitPollingRouteTest { + + @Test + public void testInstant() { + ZonedDateTime date = Instant.parse("2018-08-08T00:11:22.333Z") + .atZone(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MINUTES); + + assertEquals("2018-08-08", DATE_FORMAT.format(date)); + assertEquals("00:11", TIME_FORMAT.format(date)); + } +} diff --git a/kafka-connect-rest-source/build.gradle b/kafka-connect-rest-source/build.gradle new file mode 100644 index 00000000..f36668c1 --- /dev/null +++ b/kafka-connect-rest-source/build.gradle @@ -0,0 +1,18 @@ +dependencies { + api group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.10.0' + + // Included in connector runtime + compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion + + testImplementation group: 'ch.qos.logback', name: 'logback-classic', version:'1.2.3' + testImplementation group: 'junit', name: 'junit', version:'4.12' + testImplementation group: 'org.mockito', name: 'mockito-core', version:'1.10.19' + testImplementation group: 'com.github.tomakehurst', name: 'wiremock', version:'2.14.0' + + testImplementation group: 'org.apache.kafka', name: 'connect-api', version:'2.0.0' +} + +task copyDependencies(type: Copy) { + from configurations.runtimeClasspath.files + into "${buildDir}/third-party/" +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/AbstractRestSourceConnector.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/AbstractRestSourceConnector.java new file mode 100644 index 00000000..3c99ac89 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/AbstractRestSourceConnector.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.source.SourceConnector; +import org.radarbase.connect.rest.util.VersionUtil; + +@SuppressWarnings("unused") +public abstract class AbstractRestSourceConnector extends SourceConnector { + protected RestSourceConnectorConfig config; + + @Override + public String version() { + return VersionUtil.getVersion(); + } + + @Override + public Class taskClass() { + return RestSourceTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + return Collections.nCopies(maxTasks, new HashMap<>(config.originalsStrings())); + } + + @Override + public void start(Map props) { + config = getConfig(props); + } + + public abstract RestSourceConnectorConfig getConfig(Map conf); + + @Override + public void stop() { + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceConnectorConfig.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceConnectorConfig.java new file mode 100644 index 00000000..8007787b --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceConnectorConfig.java @@ -0,0 +1,225 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest; + +import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; + +import java.lang.reflect.InvocationTargetException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.ConfigKey; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigDef.Width; +import org.apache.kafka.connect.errors.ConnectException; +import org.radarbase.connect.rest.config.ValidClass; +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; +import org.radarbase.connect.rest.converter.StringPayloadConverter; +import org.radarbase.connect.rest.request.RequestGenerator; +import org.radarbase.connect.rest.selector.SimpleTopicSelector; +import org.radarbase.connect.rest.selector.TopicSelector; +import org.radarbase.connect.rest.single.SingleRequestGenerator; +import org.radarbase.connect.rest.util.VersionUtil; + +public class RestSourceConnectorConfig extends AbstractConfig { + public static final Pattern COLON_PATTERN = Pattern.compile(":"); + + private static final String SOURCE_POLL_INTERVAL_CONFIG = "rest.source.poll.interval.ms"; + private static final String SOURCE_POLL_INTERVAL_DOC = "How often to poll the source URL."; + private static final String SOURCE_POLL_INTERVAL_DISPLAY = "Polling interval"; + private static final Long SOURCE_POLL_INTERVAL_DEFAULT = 60000L; + + static final String SOURCE_URL_CONFIG = "rest.source.base.url"; + private static final String SOURCE_URL_DOC = "Base URL for REST source connector."; + private static final String SOURCE_URL_DISPLAY = "Base URL for REST source connector."; + + static final String SOURCE_TOPIC_SELECTOR_CONFIG = "rest.source.topic.selector"; + private static final String SOURCE_TOPIC_SELECTOR_DOC = + "The topic selector class for REST source connector."; + private static final String SOURCE_TOPIC_SELECTOR_DISPLAY = + "Topic selector class for REST source connector."; + private static final Class SOURCE_TOPIC_SELECTOR_DEFAULT = + SimpleTopicSelector.class; + + public static final String SOURCE_TOPIC_LIST_CONFIG = "rest.source.destination.topics"; + private static final String SOURCE_TOPIC_LIST_DOC = + "The list of destination topics for the REST source connector."; + private static final String SOURCE_TOPIC_LIST_DISPLAY = "Source destination topics"; + + public static final String SOURCE_PAYLOAD_CONVERTER_CONFIG = "rest.source.payload.converter.class"; + private static final Class PAYLOAD_CONVERTER_DEFAULT = + StringPayloadConverter.class; + private static final String SOURCE_PAYLOAD_CONVERTER_DOC_CONFIG = + "Class to be used to convert messages from REST calls to SourceRecords"; + private static final String SOURCE_PAYLOAD_CONVERTER_DISPLAY_CONFIG = "Payload converter class"; + + private static final String SOURCE_REQUEST_GENERATOR_CONFIG = "rest.source.request.generator.class"; + private static final Class REQUEST_GENERATOR_DEFAULT = + SingleRequestGenerator.class; + private static final String REQUEST_GENERATOR_DOC = + "Class to be used to generate REST requests"; + private static final String REQUEST_GENERATOR_DISPLAY = "Request generator class"; + + private final TopicSelector topicSelector; + private final PayloadToSourceRecordConverter payloadToSourceRecordConverter; + private final RequestGenerator requestGenerator; + + @SuppressWarnings("unchecked") + public RestSourceConnectorConfig(ConfigDef config, Map parsedConfig) { + super(config, parsedConfig); + try { + topicSelector = ((Class) + getClass(SOURCE_TOPIC_SELECTOR_CONFIG)).getDeclaredConstructor().newInstance(); + payloadToSourceRecordConverter = ((Class) + getClass(SOURCE_PAYLOAD_CONVERTER_CONFIG)).getDeclaredConstructor().newInstance(); + requestGenerator = ((Class) + getClass(SOURCE_REQUEST_GENERATOR_CONFIG)).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException + | InvocationTargetException | NoSuchMethodException e) { + throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e); + } + } + + public RestSourceConnectorConfig(Map parsedConfig) { + this(conf(), parsedConfig); + } + + public static ConfigDef conf() { + String group = "REST"; + int orderInGroup = 0; + return new ConfigDef() + .define(SOURCE_POLL_INTERVAL_CONFIG, + Type.LONG, + SOURCE_POLL_INTERVAL_DEFAULT, + Importance.LOW, + SOURCE_POLL_INTERVAL_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_POLL_INTERVAL_DISPLAY) + + .define(SOURCE_URL_CONFIG, + Type.STRING, + NO_DEFAULT_VALUE, + Importance.HIGH, + SOURCE_URL_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_URL_DISPLAY) + + .define(SOURCE_TOPIC_LIST_CONFIG, + Type.LIST, + Collections.emptyList(), + Importance.HIGH, + SOURCE_TOPIC_LIST_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_TOPIC_LIST_DISPLAY) + + .define(SOURCE_TOPIC_SELECTOR_CONFIG, + Type.CLASS, + SOURCE_TOPIC_SELECTOR_DEFAULT, + ValidClass.isSubclassOf(TopicSelector.class), + Importance.HIGH, + SOURCE_TOPIC_SELECTOR_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_TOPIC_SELECTOR_DISPLAY) + + .define(SOURCE_PAYLOAD_CONVERTER_CONFIG, + Type.CLASS, + PAYLOAD_CONVERTER_DEFAULT, + ValidClass.isSubclassOf(PayloadToSourceRecordConverter.class), + Importance.LOW, + SOURCE_PAYLOAD_CONVERTER_DOC_CONFIG, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_PAYLOAD_CONVERTER_DISPLAY_CONFIG) + + .define(SOURCE_REQUEST_GENERATOR_CONFIG, + Type.CLASS, + REQUEST_GENERATOR_DEFAULT, + ValidClass.isSubclassOf(RequestGenerator.class), + Importance.LOW, + REQUEST_GENERATOR_DOC, + group, + ++orderInGroup, + Width.SHORT, + REQUEST_GENERATOR_DISPLAY) + ; + } + + public Duration getPollInterval() { + return Duration.ofMillis(this.getLong(SOURCE_POLL_INTERVAL_CONFIG)); + } + + public String getUrl() { + return this.getString(SOURCE_URL_CONFIG); + } + + public List getTopics() { + return this.getList(SOURCE_TOPIC_LIST_CONFIG); + } + + public TopicSelector getTopicSelector() { + topicSelector.initialize(this); + return topicSelector; + } + + public PayloadToSourceRecordConverter getPayloadToSourceRecordConverter() { + payloadToSourceRecordConverter.initialize(this); + return payloadToSourceRecordConverter; + } + + private static ConfigDef getConfig() { + Map everything = new HashMap<>(conf().configKeys()); + ConfigDef visible = new ConfigDef(); + for (ConfigKey key : everything.values()) { + visible.define(key); + } + return visible; + } + + public static void main(String[] args) { + System.out.println(VersionUtil.getVersion()); + System.out.println(getConfig().toEnrichedRst()); + } + + public RequestGenerator getRequestGenerator() { + requestGenerator.initialize(this); + return requestGenerator; + } + + public long getMaxUsersPerPoll() { + return 100L; + } + + public Duration getPollIntervalPerUser() { + return Duration.ofMinutes(30); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java new file mode 100644 index 00000000..11c08d00 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java @@ -0,0 +1,93 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest; + +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.radarbase.connect.rest.request.RequestGenerator; +import org.radarbase.connect.rest.request.RestRequest; +import org.radarbase.connect.rest.util.VersionUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RestSourceTask extends SourceTask { + private static Logger logger = LoggerFactory.getLogger(RestSourceTask.class); + + private RequestGenerator requestGenerator; + + @Override + public void start(Map map) { + RestSourceConnectorConfig connectorConfig; + try { + Class connector = Class.forName(map.get("connector.class")); + connectorConfig = ((AbstractRestSourceConnector)connector.newInstance()).getConfig(map); + } catch (ClassNotFoundException e) { + throw new ConnectException("Connector " + map.get("connector.class") + " not found", e); + } catch (IllegalAccessException | InstantiationException e) { + throw new ConnectException("Connector " + map.get("connector.class") + + " could not be instantiated", e); + } + requestGenerator = connectorConfig.getRequestGenerator(); + requestGenerator.setOffsetStorageReader(context.offsetStorageReader()); + } + + @Override + public List poll() throws InterruptedException { + long timeout = MILLIS.between(Instant.now(), requestGenerator.getTimeOfNextRequest()); + if (timeout > 0) { + logger.info("Waiting {} milliseconds for next available request", timeout); + Thread.sleep(timeout); + } + + LongAdder requestsGenerated = new LongAdder(); + + List requests = requestGenerator.requests() + .peek(r -> { + logger.info("Requesting {}", r.getRequest().url()); + requestsGenerated.increment(); + }) + .flatMap(tryOrNull(RestRequest::handleRequest, + (r, ex) -> logger.warn("Failed to make request: {}", ex.toString()))) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + logger.info("Processed {} records from {} URLs", requests.size(), requestsGenerated.sum()); + + return requests; + } + + @Override + public void stop() { + logger.debug("Stopping source task"); + } + + @Override + public String version() { + return VersionUtil.getVersion(); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodRecommender.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodRecommender.java new file mode 100644 index 00000000..315d62ae --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodRecommender.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.config; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.config.ConfigDef; + +public class MethodRecommender implements ConfigDef.Recommender { + @Override + public List validValues(String name, Map connectorConfigs) { + return Arrays.asList("GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"); + } + + @Override + public boolean visible(String name, Map connectorConfigs) { + return true; + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodValidator.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodValidator.java new file mode 100644 index 00000000..e98ebded --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/MethodValidator.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.config; + +import java.util.HashMap; +import org.apache.kafka.common.config.ConfigDef; + +public class MethodValidator implements ConfigDef.Validator { + @Override + public void ensureValid(String name, Object provider) { + } + + @Override + public String toString() { + return new MethodRecommender().validValues("", new HashMap<>()).toString(); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/RestSourceTool.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/RestSourceTool.java new file mode 100644 index 00000000..df49bb33 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/RestSourceTool.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.config; + +import org.radarbase.connect.rest.RestSourceConnectorConfig; + +public interface RestSourceTool { + void initialize(RestSourceConnectorConfig config); +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/ValidClass.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/ValidClass.java new file mode 100644 index 00000000..daba5ec6 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/config/ValidClass.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.config; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; + +/** + * Validate a class name. + */ +public final class ValidClass implements ConfigDef.Validator { + private final Class superClass; + + private ValidClass(Class superClass) { + this.superClass = superClass; + } + + /** Ensures that classes are subclass of the given class and that they are instantiable. */ + public static ValidClass isSubclassOf(Class cls) { + if (cls == null) { + throw new IllegalArgumentException("Class name may not be null"); + } + return new ValidClass(cls); + } + + @Override + public void ensureValid(String name, Object obj) { + if (obj == null) { + return; + } + Class cls = (Class) obj; + if (!superClass.isAssignableFrom(cls)) { + throw new ConfigException(name, obj, + "Class " + obj + " must be subclass of " + superClass.getName()); + } + try { + cls.newInstance(); + } catch (InstantiationException ex) { + throw new ConfigException(name, obj, "Class " + obj + " must be instantiable: " + ex); + } catch (IllegalAccessException ex) { + throw new ConfigException(name, obj, "Class " + obj + " must be accessible: " + ex); + } + } + + public String toString() { + return "Class extending " + superClass.getName(); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/BytesPayloadConverter.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/BytesPayloadConverter.java new file mode 100644 index 00000000..3448079b --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/BytesPayloadConverter.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.converter; + +import static java.lang.System.currentTimeMillis; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.request.RestRequest; +import org.radarbase.connect.rest.selector.TopicSelector; + +public class BytesPayloadConverter implements PayloadToSourceRecordConverter { + private TopicSelector topicSelector; + + // Just bytes for incoming messages + @Override + public Collection convert(RestRequest request, Response response) throws IOException { + Map sourceOffset = Collections.singletonMap( + TIMESTAMP_OFFSET_KEY, currentTimeMillis()); + ResponseBody body = response.body(); + byte[] result = body != null ? body.bytes() : null; + String topic = topicSelector.getTopic(request, result); + return Collections.singleton( + new SourceRecord(request.getPartition(), sourceOffset, + topic, Schema.BYTES_SCHEMA, result)); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + topicSelector = config.getTopicSelector(); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/PayloadToSourceRecordConverter.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/PayloadToSourceRecordConverter.java new file mode 100644 index 00000000..d0f5cdc5 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/PayloadToSourceRecordConverter.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.converter; + +import java.io.IOException; +import java.util.Collection; +import okhttp3.Response; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.config.RestSourceTool; +import org.radarbase.connect.rest.request.RestRequest; + +public interface PayloadToSourceRecordConverter extends RestSourceTool { + String TIMESTAMP_OFFSET_KEY = "timestamp"; + + Collection convert( + RestRequest request, Response response) throws IOException; +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/StringPayloadConverter.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/StringPayloadConverter.java new file mode 100644 index 00000000..f9e36750 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/converter/StringPayloadConverter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.converter; + +import static java.lang.System.currentTimeMillis; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.request.RestRequest; +import org.radarbase.connect.rest.selector.TopicSelector; + +public class StringPayloadConverter implements PayloadToSourceRecordConverter { + private TopicSelector topicSelector; + + @Override + public Collection convert(RestRequest request, Response response) throws IOException { + Map sourceOffset = Collections.singletonMap(TIMESTAMP_OFFSET_KEY, currentTimeMillis()); + ResponseBody body = response.body(); + String result = body == null ? null : body.string(); + String topic = topicSelector.getTopic(request, result); + return Collections.singleton( + new SourceRecord(request.getPartition(), sourceOffset, topic, + Schema.STRING_SCHEMA, result)); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + topicSelector = config.getTopicSelector(); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/PollingRequestRoute.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/PollingRequestRoute.java new file mode 100644 index 00000000..dfea4822 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/PollingRequestRoute.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.request; + +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.stream.Stream; + +public interface PollingRequestRoute extends RequestRoute { + /** General polling interval for retrying this route. */ + Duration getPollInterval(); + /** Last time the route was polled. */ + Instant getLastPoll(); + /** Actual times that new data will be needed. */ + Stream nextPolls(); + + /** Get the time that this route should be polled again. */ + default Instant getTimeOfNextRequest() { + return max(getLastPoll().plus(getPollInterval()), + nextPolls().min(Comparator.naturalOrder()).orElse(Instant.MAX)); + } + + static > T max(T a, T b) { + return a != null && (b == null || a.compareTo(b) >= 0) ? a : b; + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGenerator.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGenerator.java new file mode 100644 index 00000000..70514493 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGenerator.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.request; + +import java.time.Instant; +import java.util.stream.Stream; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.config.RestSourceTool; + +/** + * Dynamically generates requests. The requests should be based on the offsets that are stored in + * the response SourceRecord. + */ +public interface RequestGenerator extends RestSourceTool { + Instant getTimeOfNextRequest(); + + /** + * Requests that should be queried next. + */ + Stream requests(); + + /** + * Set the Kafka offset storage reader. This allows for resetting the request intervals. + * @param offsetStorageReader possibly null offset storage reader. + */ + void setOffsetStorageReader(OffsetStorageReader offsetStorageReader); +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGeneratorRouter.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGeneratorRouter.java new file mode 100644 index 00000000..72ac4a76 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestGeneratorRouter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.request; + +import java.time.Instant; +import java.util.Comparator; +import java.util.stream.Stream; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.RestSourceConnectorConfig; + +public abstract class RequestGeneratorRouter implements RequestGenerator { + @Override + public Stream requests() { + return routes() + .flatMap(RequestRoute::requests); + } + + @Override + public Instant getTimeOfNextRequest() { + return routes() + .map(RequestRoute::getTimeOfNextRequest) + .min(Comparator.naturalOrder()) + .orElse(Instant.MAX); + } + + public abstract Stream routes(); + + public void setOffsetStorageReader(OffsetStorageReader offsetStorageReader) { + routes().forEach(r -> r.setOffsetStorageReader(offsetStorageReader)); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + routes().forEach(r -> r.initialize(config)); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestRoute.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestRoute.java new file mode 100644 index 00000000..fb5b007b --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RequestRoute.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.request; + +import okhttp3.Response; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; + +/** Single request route. This may represent e.g. a URL. */ +public interface RequestRoute extends RequestGenerator { + /** Data converter from data that is returned from the route. */ + PayloadToSourceRecordConverter converter(); + + /** + * Called when the request from this route succeeded. + * + * @param request non-null generated request + * @param record non-null resulting record + */ + void requestSucceeded(RestRequest request, SourceRecord record); + void requestEmpty(RestRequest request); + void requestFailed(RestRequest request, Response response); +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java new file mode 100644 index 00000000..8e00b6da --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.request; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Stream; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.apache.kafka.connect.source.SourceRecord; + +/** + * Single request. This must originate from a RequestRoute and have a predefined source partition. + */ +public class RestRequest { + private final Request request; + private final Map partition; + private final RequestRoute route; + private final OkHttpClient client; + + /** + * Single RestRequest. + * @param route originating route + * @param client OkHttp client to make request with + * @param request OkHttp request to make + * @param partition Kafka source partition + */ + public RestRequest( + RequestRoute route, + OkHttpClient client, + Request request, + Map partition) { + this.request = request; + this.partition = partition; + this.route = route; + this.client = client; + } + + public Request getRequest() { + return request; + } + + public Map getPartition() { + return partition; + } + + /** + * Handle the request using the internal client, using the request route converter. + * @return stream of resulting source records, or {@code null} if the response was not successful. + * @throws IOException if making or parsing the request failed. + */ + public Stream handleRequest() throws IOException { + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + route.requestFailed(this, response); + return null; + } + + Collection records = route.converter().convert(this, response); + if (records.isEmpty()) { + route.requestEmpty(this); + } else { + records.forEach(r -> route.requestSucceeded(this, r)); + } + return records.stream(); + } catch (IOException ex) { + route.requestFailed(this, null); + throw ex; + } + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/SimpleTopicSelector.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/SimpleTopicSelector.java new file mode 100644 index 00000000..9dfeee77 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/SimpleTopicSelector.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.selector; + +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.request.RestRequest; + +public class SimpleTopicSelector implements TopicSelector { + private String topic; + + @Override + public String getTopic(RestRequest request, Object result) { + return topic; + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + // Always return the first topic in the list + topic = config.getTopics().get(0); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/TopicSelector.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/TopicSelector.java new file mode 100644 index 00000000..60f104a4 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/selector/TopicSelector.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.selector; + +import org.radarbase.connect.rest.config.RestSourceTool; +import org.radarbase.connect.rest.request.RestRequest; + +public interface TopicSelector extends RestSourceTool { + String getTopic(RestRequest request, Object result); +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java new file mode 100644 index 00000000..d2fd11ab --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java @@ -0,0 +1,138 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.single; + +import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.TIMESTAMP_OFFSET_KEY; +import static org.radarbase.connect.rest.request.PollingRequestRoute.max; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Stream; +import okhttp3.Headers; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; +import org.radarbase.connect.rest.request.RequestRoute; +import org.radarbase.connect.rest.request.RestRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Loads a single URL. */ +public class SingleRequestGenerator implements RequestRoute { + private static final Logger logger = LoggerFactory.getLogger(SingleRequestGenerator.class); + + private Instant lastTimestamp; + private HttpUrl url; + private String method; + private RequestBody body; + private Duration pollInterval; + private Instant lastPoll; + private Map key; + private Headers headers; + private PayloadToSourceRecordConverter converter; + private OkHttpClient client; + + @Override + public void initialize(RestSourceConnectorConfig config) { + SingleRestSourceConnectorConfig singleConfig = (SingleRestSourceConnectorConfig) config; + this.pollInterval = config.getPollInterval(); + lastPoll = Instant.MIN; + + this.url = HttpUrl.parse(config.getUrl()); + this.key = Collections.singletonMap("URL", config.getUrl()); + this.method = singleConfig.getMethod(); + + Headers.Builder headersBuilder = new Headers.Builder(); + for (Map.Entry header : singleConfig.getRequestProperties().entrySet()) { + headersBuilder.add(header.getKey(), header.getValue()); + } + this.headers = headersBuilder.build(); + + if (singleConfig.getData() != null && !singleConfig.getData().isEmpty()) { + String contentType = headers.get("Content-Type"); + MediaType mediaType; + if (contentType == null) { + mediaType = MediaType.parse("text/plain; charset=utf-8"); + } else { + mediaType = MediaType.parse(contentType); + } + body = RequestBody.create(mediaType, singleConfig.getData()); + } + + converter = config.getPayloadToSourceRecordConverter(); + client = new OkHttpClient(); + } + + @Override + public Instant getTimeOfNextRequest() { + return max(lastTimestamp, lastPoll).plus(pollInterval); + } + + @Override + public Stream requests() { + return Stream.of(new RestRequest(this, client, new Request.Builder() + .method(method, body) + .url(url) + .headers(headers) + .build(), key)); + } + + @Override + public void requestSucceeded(RestRequest processedResponse, SourceRecord record) { + lastTimestamp = Instant.ofEpochMilli((Long)record.sourceOffset().get(TIMESTAMP_OFFSET_KEY)); + lastPoll = Instant.now(); + } + + @Override + public void requestEmpty(RestRequest request) { + lastPoll = Instant.now(); + } + + @Override + public void requestFailed(RestRequest request, Response response) { + lastPoll = Instant.now(); + } + + @Override + public void setOffsetStorageReader(OffsetStorageReader offsetStorageReader) { + lastTimestamp = Instant.MIN; + + if (offsetStorageReader != null) { + Map offset = offsetStorageReader.offset(key); + if (offset != null) { + lastTimestamp = Instant.ofEpochMilli((Long) offset.getOrDefault(TIMESTAMP_OFFSET_KEY, 0L)); + } + } else { + logger.warn("Offset storage reader is not provided. Cannot restart from previous timestamp."); + } + } + + @Override + public PayloadToSourceRecordConverter converter() { + return converter; + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnector.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnector.java new file mode 100644 index 00000000..0f053812 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnector.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.single; + +import java.util.Map; +import org.apache.kafka.common.config.ConfigDef; +import org.radarbase.connect.rest.AbstractRestSourceConnector; +import org.radarbase.connect.rest.RestSourceConnectorConfig; + +public class SingleRestSourceConnector extends AbstractRestSourceConnector { + @Override + public ConfigDef config() { + return SingleRestSourceConnectorConfig.conf(); + } + + @Override + public RestSourceConnectorConfig getConfig(Map conf) { + return new SingleRestSourceConnectorConfig(conf); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnectorConfig.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnectorConfig.java new file mode 100644 index 00000000..a056b4b3 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRestSourceConnectorConfig.java @@ -0,0 +1,119 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.single; + +import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.config.MethodRecommender; +import org.radarbase.connect.rest.config.MethodValidator; + +public class SingleRestSourceConnectorConfig extends RestSourceConnectorConfig { + public static final String SOURCE_METHOD_CONFIG = "rest.source.method"; + private static final String SOURCE_METHOD_DOC = "The HTTP method for REST source connector."; + private static final String SOURCE_METHOD_DISPLAY = "Source method"; + private static final String SOURCE_METHOD_DEFAULT = "POST"; + + public static final String SOURCE_PROPERTIES_LIST_CONFIG = "rest.source.properties"; + private static final String SOURCE_PROPERTIES_LIST_DOC = + "The request properties (headers) for REST source connector."; + private static final String SOURCE_PROPERTIES_LIST_DISPLAY = "Source properties"; + + public static final String SOURCE_DATA_CONFIG = "rest.source.data"; + private static final String SOURCE_DATA_DOC = "The data for REST source connector."; + private static final String SOURCE_DATA_DISPLAY = "Data for REST source connector."; + private static final String SOURCE_DATA_DEFAULT = null; + + private final Map requestProperties; + + @SuppressWarnings("unchecked") + private SingleRestSourceConnectorConfig(ConfigDef config, Map parsedConfig) { + super(config, parsedConfig); + requestProperties = getPropertiesList().stream() + .map(COLON_PATTERN::split) + .collect(Collectors.toMap(a -> a[0], a -> a[1])); + } + + public SingleRestSourceConnectorConfig(Map parsedConfig) { + this(SingleRestSourceConnectorConfig.conf(), parsedConfig); + } + + public static ConfigDef conf() { + String group = "Single REST source"; + ConfigDef superConf = RestSourceConnectorConfig.conf(); + int orderInGroup = superConf.names().size(); + return superConf + .define(SOURCE_METHOD_CONFIG, + Type.STRING, + SOURCE_METHOD_DEFAULT, + new MethodValidator(), + Importance.HIGH, + SOURCE_METHOD_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + SOURCE_METHOD_DISPLAY, + new MethodRecommender()) + + .define(SOURCE_PROPERTIES_LIST_CONFIG, + Type.LIST, + NO_DEFAULT_VALUE, + Importance.HIGH, + SOURCE_PROPERTIES_LIST_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + SOURCE_PROPERTIES_LIST_DISPLAY) + + .define(SOURCE_DATA_CONFIG, + Type.STRING, + SOURCE_DATA_DEFAULT, + Importance.LOW, + SOURCE_DATA_DOC, + group, + ++orderInGroup, + ConfigDef.Width.SHORT, + SOURCE_DATA_DISPLAY); + } + + public List getPropertiesList() { + return this.getList(SOURCE_PROPERTIES_LIST_CONFIG); + } + + public String getMethod() { + return this.getString(SOURCE_METHOD_CONFIG); + } + + public String getData() { + return this.getString(SOURCE_DATA_CONFIG); + } + + public Map getRequestProperties() { + return requestProperties; + } + + public static void main(String[] args) { + System.out.println(conf().toHtmlTable()); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/ThrowingFunction.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/ThrowingFunction.java new file mode 100644 index 00000000..60ad9c92 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/ThrowingFunction.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.util; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; + +@FunctionalInterface +public interface ThrowingFunction { + R apply(T value) throws Exception; + + static Function tryOrNull(ThrowingFunction tryClause, BiConsumer catchClause) { + return t -> { + try { + return tryClause.apply(t); + } catch (Exception e) { + catchClause.accept(t, e); + return null; + } + }; + } + + static Function tryOrRethrow(ThrowingFunction tryClause, BiFunction catchClause) { + return t -> { + try { + return tryClause.apply(t); + } catch (Exception e) { + throw catchClause.apply(t, e); + } + }; + } + + static Function tryOrRethrow(ThrowingFunction tryClause) { + return tryOrRethrow(tryClause, (t, ex) -> { + if (ex instanceof IOException) { + throw new UncheckedIOException((IOException) ex); + } else if (ex instanceof RuntimeException) { + throw (RuntimeException) ex; + } else { + throw new RuntimeException(ex); + } + }); + } +} diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/VersionUtil.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/VersionUtil.java new file mode 100644 index 00000000..0d403db6 --- /dev/null +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/util/VersionUtil.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.util; + +public final class VersionUtil { + private VersionUtil() { + // utility class + } + + public static String getVersion() { + try { + return VersionUtil.class.getPackage().getImplementationVersion(); + } catch (Exception ex) { + return "0.0.0.0"; + } + } +} diff --git a/kafka-connect-rest-source/src/test/java/org/radarbase/connect/rest/RestTaskTest.java b/kafka-connect-rest-source/src/test/java/org/radarbase/connect/rest/RestTaskTest.java new file mode 100644 index 00000000..a9717fa2 --- /dev/null +++ b/kafka-connect-rest-source/src/test/java/org/radarbase/connect/rest/RestTaskTest.java @@ -0,0 +1,151 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.junit.Assert.assertEquals; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import java.net.ServerSocket; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTaskContext; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.junit.Rule; +import org.junit.Test; +import org.radarbase.connect.rest.converter.BytesPayloadConverter; +import org.radarbase.connect.rest.converter.StringPayloadConverter; +import org.radarbase.connect.rest.selector.SimpleTopicSelector; +import org.radarbase.connect.rest.single.SingleRestSourceConnector; +import org.radarbase.connect.rest.single.SingleRestSourceConnectorConfig; + +public class RestTaskTest { + + private static final String CONTENT_TYPE = "Content-Type"; + private static final String ACCEPT = "Accept"; + private static final String APPLICATION_JSON = "application/json; charset=UTF-8"; + + private static final String TOPIC = "rest-source-destination-topic"; + private static final String REST_SOURCE_DESTINATION_TOPIC_LIST = TOPIC; + + private static final String METHOD = "POST"; + private static final String PROPERTIES_LIST = "" + + CONTENT_TYPE + ":" + APPLICATION_JSON + ", " + + ACCEPT + ":" + APPLICATION_JSON; + private static final String TOPIC_SELECTOR = SimpleTopicSelector.class.getName(); + private static final String BYTES_PAYLOAD_CONVERTER = BytesPayloadConverter.class.getName(); + private static final String STRING_PAYLOAD_CONVERTER = StringPayloadConverter.class.getName(); + private static final String DATA = "{\"A\":\"B\"}"; + private static final String RESPONSE_BODY = "{\"B\":\"A\"}"; + private static final int PORT = getPort(); + private static final String PATH = "/my/resource"; + private static final String URL = "http://localhost:" + PORT + PATH; + + @Rule + public WireMockRule wireMockRule = new WireMockRule(PORT); + + @Test + public void restTest() throws InterruptedException { + stubFor(post(urlEqualTo(PATH)) + .withHeader(ACCEPT, equalTo(APPLICATION_JSON)) + .willReturn(aResponse() + .withStatus(200) + .withHeader(CONTENT_TYPE, APPLICATION_JSON) + .withBody(RESPONSE_BODY))); + + Map props; + props = new HashMap<>(); + props.put("connector.class", SingleRestSourceConnector.class.getName()); + props.put(SingleRestSourceConnectorConfig.SOURCE_METHOD_CONFIG, METHOD); + props.put(SingleRestSourceConnectorConfig.SOURCE_PROPERTIES_LIST_CONFIG, PROPERTIES_LIST); + props.put(RestSourceConnectorConfig.SOURCE_URL_CONFIG, URL); + props.put(SingleRestSourceConnectorConfig.SOURCE_DATA_CONFIG, DATA); + props.put(RestSourceConnectorConfig.SOURCE_TOPIC_SELECTOR_CONFIG, TOPIC_SELECTOR); + props.put(RestSourceConnectorConfig.SOURCE_TOPIC_LIST_CONFIG, REST_SOURCE_DESTINATION_TOPIC_LIST); + props.put(RestSourceConnectorConfig.SOURCE_PAYLOAD_CONVERTER_CONFIG, STRING_PAYLOAD_CONVERTER); + + + RestSourceTask sourceTask; + List messages; + + sourceTask = new RestSourceTask(); + SourceTaskContext context = new SourceTaskContext() { + @Override + public Map configs() { + return props; + } + + @Override + public OffsetStorageReader offsetStorageReader() { + return null; + } + }; + + sourceTask.initialize(context); + sourceTask.start(props); + messages = sourceTask.poll(); + + assertEquals("Message count: ", 1, messages.size()); + assertEquals("Response class: ", String.class, messages.get(0).value().getClass()); + assertEquals("Response body: ", RESPONSE_BODY, messages.get(0).value()); + assertEquals("Topic: ", TOPIC, messages.get(0).topic()); + + verify(postRequestedFor(urlMatching(PATH)) + .withRequestBody(equalTo(DATA)) + .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); + + props.put(RestSourceConnectorConfig.SOURCE_PAYLOAD_CONVERTER_CONFIG, BYTES_PAYLOAD_CONVERTER); + + sourceTask = new RestSourceTask(); + sourceTask.initialize(context); + sourceTask.start(props); + messages = sourceTask.poll(); + + assertEquals("Message count: ", 1, messages.size()); + assertEquals("Response class: ", byte[].class, messages.get(0).value().getClass()); + assertEquals("Response body: ", RESPONSE_BODY, new String((byte[]) messages.get(0).value())); + assertEquals("Topic: ", TOPIC, messages.get(0).topic()); + + verify(postRequestedFor(urlMatching(PATH)) + .withRequestBody(equalTo(DATA)) + .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); + + wireMockRule.resetRequests(); + } + + private static int getPort() { + try { + ServerSocket s = new ServerSocket(0); + int localPort = s.getLocalPort(); + s.close(); + return localPort; + } catch (Exception e) { + throw new RuntimeException("Failed to get a free PORT", e); + } + } +} diff --git a/src/test/resources/logback.xml b/kafka-connect-rest-source/src/test/resources/logback.xml similarity index 100% rename from src/test/resources/logback.xml rename to kafka-connect-rest-source/src/test/resources/logback.xml diff --git a/pom.xml b/pom.xml deleted file mode 100644 index ea5a42bd..00000000 --- a/pom.xml +++ /dev/null @@ -1,150 +0,0 @@ - - 4.0.0 - - com.tm.kafka - kafka-connect-rest - 1.0-SNAPSHOT - jar - - kafka-connect-rest - A Kafka Connect Connector for kafka-connect-rest - - - UTF-8 - 1.1.0 - 2.9.5 - 4.1.1 - http://packages.confluent.io/maven/ - - - - - - org.apache.kafka - connect-api - ${kafka.version} - provided - - - - org.apache.kafka - connect-json - ${kafka.version} - provided - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - provided - - - - org.apache.commons - commons-compress - 1.15 - - - - org.apache.velocity - velocity-engine-core - 2.0 - - - - ch.qos.logback - logback-classic - 1.2.3 - test - - - - junit - junit - 4.12 - test - - - - org.mockito - mockito-core - 1.10.19 - test - - - - com.github.tomakehurst - wiremock - 2.14.0 - test - - - - - - - confluent - Confluent - ${confluent.maven.repo} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.0.2 - - - - true - true - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.5.1 - true - - 1.8 - 1.8 - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.1.0 - - true - false - - - - package - - shade - - - - - - - - - - - - - - - - - diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000..c2ce6f87 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,4 @@ +rootProject.name = 'kafka-connect-rest-source' +include ':kafka-connect-fitbit-source' +include ':kafka-connect-rest-source' + diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSinkConnector.java b/src/main/java/com/tm/kafka/connect/rest/RestSinkConnector.java deleted file mode 100644 index f20383a9..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSinkConnector.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.tm.kafka.connect.rest; - -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.sink.SinkConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class RestSinkConnector extends SinkConnector { - private static Logger log = LoggerFactory.getLogger(RestSinkConnector.class); - private RestSinkConnectorConfig config; - - @Override - public String version() { - return VersionUtil.getVersion(); - } - - @Override - public void start(Map map) { - config = new RestSinkConnectorConfig(map); - } - - @Override - public Class taskClass() { - return RestSinkTask.class; - } - - @Override - public List> taskConfigs(int maxTasks) { - Map taskProps = new HashMap<>(config.originalsStrings()); - List> taskConfigs = new ArrayList<>(maxTasks); - for (int i = 0; i < maxTasks; ++i) { - taskConfigs.add(taskProps); - } - return taskConfigs; - } - - @Override - public void stop() { - } - - @Override - public ConfigDef config() { - return RestSinkConnectorConfig.conf(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSinkConnectorConfig.java b/src/main/java/com/tm/kafka/connect/rest/RestSinkConnectorConfig.java deleted file mode 100644 index ddecf1f4..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSinkConnectorConfig.java +++ /dev/null @@ -1,279 +0,0 @@ -package com.tm.kafka.connect.rest; - -import com.tm.kafka.connect.rest.converter.BytesPayloadConverter; -import com.tm.kafka.connect.rest.converter.SinkRecordToPayloadConverter; -import com.tm.kafka.connect.rest.converter.StringPayloadConverter; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.connect.errors.ConnectException; - -import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; - -public class RestSinkConnectorConfig extends AbstractConfig { - static final String SINK_METHOD_CONFIG = "rest.sink.method"; - private static final String SINK_METHOD_DOC = "The HTTP method for REST sink connector."; - private static final String SINK_METHOD_DISPLAY = "Sink method"; - private static final String SINK_METHOD_DEFAULT = "POST"; - - static final String SINK_PROPERTIES_LIST_CONFIG = "rest.sink.properties"; - private static final String SINK_PROPERTIES_LIST_DOC = - "The request properties (headers) for REST sink connector."; - private static final String SINK_PROPERTIES_LIST_DISPLAY = "Sink properties"; - - static final String SINK_URL_CONFIG = "rest.sink.url"; - private static final String SINK_URL_DOC = "The URL for REST sink connector."; - private static final String SINK_URL_DISPLAY = "URL for REST sink connector."; - - static final String SINK_PAYLOAD_CONVERTER_CONFIG = "rest.sink.payload.converter.class"; - private static final Class PAYLOAD_CONVERTER_DEFAULT = - StringPayloadConverter.class; - private static final String SINK_PAYLOAD_CONVERTER_DOC = - "Class to be used to convert messages from SinkRecords to Strings for REST calls"; - private static final String SINK_PAYLOAD_CONVERTER_DISPLAY = "Payload converter class"; - - private static final String SINK_PAYLOAD_CONVERTER_SCHEMA_CONFIG = "rest.sink.payload.converter.schema"; - private static final String SINK_PAYLOAD_CONVERTER_SCHEMA_DOC = "Include schema in JSON output for JsonPayloadConverter"; - private static final String SINK_PAYLOAD_CONVERTER_SCHEMA_DISPLAY = "Include schema in JSON output (true/false)"; - private static final String SINK_PAYLOAD_CONVERTER_SCHEMA_DEFAULT = "false"; - - private static final String SINK_RETRY_BACKOFF_CONFIG = "rest.sink.retry.backoff.ms"; - private static final String SINK_RETRY_BACKOFF_DOC = - "The retry backoff in milliseconds. This config is used to notify Kafka connect to retry " - + "delivering a message batch or performing recovery in case of transient exceptions."; - private static final long SINK_RETRY_BACKOFF_DEFAULT = 5000L; - private static final String SINK_RETRY_BACKOFF_DISPLAY = "Retry Backoff (ms)"; - - private static final String SINK_VELOCITY_TEMPLATE_CONFIG = "rest.sink.velocity.template"; - private static final String SINK_VELOCITY_TEMPLATE_DOC = - "Velocity template file to convert incoming messages to be used in a REST call."; - private static final String SINK_VELOCITY_TEMPLATE_DEFAULT = "rest.vm"; - private static final String SINK_VELOCITY_TEMPLATE_DISPLAY = "Velocity template"; - - private final SinkRecordToPayloadConverter sinkRecordToPayloadConverter; - private final Map requestProperties; - - @SuppressWarnings("unchecked") - private RestSinkConnectorConfig(ConfigDef config, Map parsedConfig) { - super(config, parsedConfig); - try { - sinkRecordToPayloadConverter = ((Class) - getClass(SINK_PAYLOAD_CONVERTER_CONFIG)).getDeclaredConstructor().newInstance(); - } catch (IllegalAccessException | InstantiationException - | InvocationTargetException | NoSuchMethodException e) { - throw new ConnectException("Invalid class for: " + SINK_PAYLOAD_CONVERTER_CONFIG, e); - } - requestProperties = getPropertiesList().stream() - .map(a -> a.split(":")) - .collect(Collectors.toMap(a -> a[0], a -> a[1])); - } - - public RestSinkConnectorConfig(Map parsedConfig) { - this(conf(), parsedConfig); - } - - static ConfigDef conf() { - String group = "REST"; - int orderInGroup = 0; - return new ConfigDef() - .define(SINK_METHOD_CONFIG, - Type.STRING, - SINK_METHOD_DEFAULT, - new MethodValidator(), - Importance.HIGH, - SINK_METHOD_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SINK_METHOD_DISPLAY, - new MethodRecommender()) - - .define(SINK_PROPERTIES_LIST_CONFIG, - Type.LIST, - NO_DEFAULT_VALUE, - Importance.HIGH, - SINK_PROPERTIES_LIST_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SINK_PROPERTIES_LIST_DISPLAY) - - .define(SINK_URL_CONFIG, - Type.STRING, - NO_DEFAULT_VALUE, - Importance.HIGH, - SINK_URL_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SINK_URL_DISPLAY) - - .define(SINK_PAYLOAD_CONVERTER_CONFIG, - Type.CLASS, - PAYLOAD_CONVERTER_DEFAULT, - new PayloadConverterValidator(), - Importance.LOW, - SINK_PAYLOAD_CONVERTER_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SINK_PAYLOAD_CONVERTER_DISPLAY, - new PayloadConverterRecommender()) - - .define(SINK_PAYLOAD_CONVERTER_SCHEMA_CONFIG, - Type.BOOLEAN, - SINK_PAYLOAD_CONVERTER_SCHEMA_DEFAULT, - new PayloadConverterSchemaValidator(), - Importance.LOW, - SINK_PAYLOAD_CONVERTER_SCHEMA_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SINK_PAYLOAD_CONVERTER_SCHEMA_DISPLAY - ) - - .define(SINK_RETRY_BACKOFF_CONFIG, - Type.LONG, - SINK_RETRY_BACKOFF_DEFAULT, - Importance.LOW, - SINK_RETRY_BACKOFF_DOC, - group, - ++orderInGroup, - ConfigDef.Width.NONE, - SINK_RETRY_BACKOFF_DISPLAY) - - .define(SINK_VELOCITY_TEMPLATE_CONFIG, - Type.STRING, - SINK_VELOCITY_TEMPLATE_DEFAULT, - Importance.LOW, - SINK_VELOCITY_TEMPLATE_DOC, - group, - ++orderInGroup, - ConfigDef.Width.NONE, - SINK_VELOCITY_TEMPLATE_DISPLAY) - ; - } - - String getMethod() { - return this.getString(SINK_METHOD_CONFIG); - } - - private List getPropertiesList() { - return this.getList(SINK_PROPERTIES_LIST_CONFIG); - } - - public String getUrl() { - return this.getString(SINK_URL_CONFIG); - } - - Long getRetryBackoff() { - return this.getLong(SINK_RETRY_BACKOFF_CONFIG); - } - - public Boolean getIncludeSchema() { - return this.getBoolean(SINK_PAYLOAD_CONVERTER_SCHEMA_CONFIG); - } - - SinkRecordToPayloadConverter getSinkRecordToPayloadConverter() { - return sinkRecordToPayloadConverter; - } - - Map getRequestProperties() { - return requestProperties; - } - - public String getVelocityTemplate() { - return this.getString(SINK_VELOCITY_TEMPLATE_CONFIG); - } - - private static class PayloadConverterRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Arrays.asList(StringPayloadConverter.class, BytesPayloadConverter.class); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } - } - - private static class PayloadConverterValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object provider) { - if (provider instanceof Class - && SinkRecordToPayloadConverter.class.isAssignableFrom((Class) provider)) { - return; - } - throw new ConfigException(name, provider, "Class must extend: " - + SinkRecordToPayloadConverter.class); - } - - @Override - public String toString() { - return "Any class implementing: " + SinkRecordToPayloadConverter.class; - } - } - - private static class PayloadConverterSchemaRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Arrays.asList(Boolean.TRUE.toString(), Boolean.FALSE.toString()); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } - } - - private static class PayloadConverterSchemaValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object provider) { - if (provider instanceof Boolean) { - Boolean value = (Boolean) provider; - if (value.equals(true) || (value.equals(false))) { - return; - } - } - throw new ConfigException(name, provider, "Please provide 'true' or 'false'"); - } - - @Override - public String toString() { - return new PayloadConverterSchemaRecommender().validValues("", new HashMap<>()).toString(); - } - } - - private static class MethodRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Arrays.asList("GET", "POST"); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } - } - - private static class MethodValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object provider) { - } - - @Override - public String toString() { - return new MethodRecommender().validValues("", new HashMap<>()).toString(); - } - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSinkTask.java b/src/main/java/com/tm/kafka/connect/rest/RestSinkTask.java deleted file mode 100644 index ccf27a77..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSinkTask.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.tm.kafka.connect.rest; - -import com.tm.kafka.connect.rest.converter.SinkRecordToPayloadConverter; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.util.Collection; -import java.util.Map; - -public class RestSinkTask extends SinkTask { - private static Logger log = LoggerFactory.getLogger(RestSinkTask.class); - - private String method; - private Map requestProperties; - private String url; - private SinkRecordToPayloadConverter converter; - private Long retryBackoff; - - @Override - public void start(Map map) { - RestSinkConnectorConfig connectorConfig = new RestSinkConnectorConfig(map); - retryBackoff = connectorConfig.getRetryBackoff(); - method = connectorConfig.getMethod(); - requestProperties = connectorConfig.getRequestProperties(); - url = connectorConfig.getUrl(); - converter = connectorConfig.getSinkRecordToPayloadConverter(); - converter.start(connectorConfig); - } - - @Override - public void put(Collection records) { - for (SinkRecord record : records) { - while (true) { - try { - String data = converter.convert(record); - String u = url; - if ("GET".equals(method)) { - u = u + URLEncoder.encode(data, "UTF-8"); - } - HttpURLConnection conn = (HttpURLConnection) new URL(u).openConnection(); - requestProperties.forEach(conn::setRequestProperty); - conn.setRequestMethod(method); - if ("POST".equals(method)) { - conn.setDoOutput(true); - OutputStream os = conn.getOutputStream(); - os.write(data.getBytes()); - os.flush(); - } - int responseCode = conn.getResponseCode(); - if (log.isTraceEnabled()) { - log.trace("Response code: {}, Request data: {}", responseCode, data); - } - break; - } catch (Exception e) { - log.error("HTTP call failed", e); - try { - Thread.sleep(retryBackoff); - } catch (Exception ignored) { - // Ignored - } - } - } - } - } - - @Override - public void stop() { - log.debug("Stopping sink task, setting client to null"); - } - - @Override - public String version() { - return VersionUtil.getVersion(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSourceConnector.java b/src/main/java/com/tm/kafka/connect/rest/RestSourceConnector.java deleted file mode 100644 index b2c431f6..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSourceConnector.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.tm.kafka.connect.rest; - -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.source.SourceConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class RestSourceConnector extends SourceConnector { - private static Logger log = LoggerFactory.getLogger(RestSourceConnector.class); - private RestSourceConnectorConfig config; - - @Override - public String version() { - return VersionUtil.getVersion(); - } - - @Override - public void start(Map map) { - config = new RestSourceConnectorConfig(map); - } - - @Override - public Class taskClass() { - return RestSourceTask.class; - } - - @Override - public List> taskConfigs(int maxTasks) { - Map taskProps = new HashMap<>(config.originalsStrings()); - List> taskConfigs = new ArrayList<>(maxTasks); - for (int i = 0; i < maxTasks; ++i) { - taskConfigs.add(taskProps); - } - return taskConfigs; - } - - @Override - public void stop() { - } - - @Override - public ConfigDef config() { - return RestSourceConnectorConfig.conf(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSourceConnectorConfig.java b/src/main/java/com/tm/kafka/connect/rest/RestSourceConnectorConfig.java deleted file mode 100644 index e84d476b..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSourceConnectorConfig.java +++ /dev/null @@ -1,240 +0,0 @@ -package com.tm.kafka.connect.rest; - -import com.tm.kafka.connect.rest.config.MethodRecommender; -import com.tm.kafka.connect.rest.config.MethodValidator; -import com.tm.kafka.connect.rest.config.PayloadToSourceRecordConverterRecommender; -import com.tm.kafka.connect.rest.config.PayloadToSourceRecordConverterValidator; -import com.tm.kafka.connect.rest.config.TopicSelectorRecommender; -import com.tm.kafka.connect.rest.config.TopicSelectorValidator; -import com.tm.kafka.connect.rest.converter.PayloadToSourceRecordConverter; -import com.tm.kafka.connect.rest.converter.StringPayloadConverter; -import com.tm.kafka.connect.rest.selector.SimpleTopicSelector; -import com.tm.kafka.connect.rest.selector.TopicSelector; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; -import org.apache.kafka.connect.errors.ConnectException; - -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; - -public class RestSourceConnectorConfig extends AbstractConfig { - - private static final String SOURCE_POLL_INTERVAL_CONFIG = "rest.source.poll.interval.ms"; - private static final String SOURCE_POLL_INTERVAL_DOC = "How often to poll the source URL."; - private static final String SOURCE_POLL_INTERVAL_DISPLAY = "Polling interval"; - private static final Long SOURCE_POLL_INTERVAL_DEFAULT = 60000L; - - static final String SOURCE_METHOD_CONFIG = "rest.source.method"; - private static final String SOURCE_METHOD_DOC = "The HTTP method for REST source connector."; - private static final String SOURCE_METHOD_DISPLAY = "Source method"; - private static final String SOURCE_METHOD_DEFAULT = "POST"; - - static final String SOURCE_PROPERTIES_LIST_CONFIG = "rest.source.properties"; - private static final String SOURCE_PROPERTIES_LIST_DOC = - "The request properties (headers) for REST source connector."; - private static final String SOURCE_PROPERTIES_LIST_DISPLAY = "Source properties"; - - static final String SOURCE_URL_CONFIG = "rest.source.url"; - private static final String SOURCE_URL_DOC = "The URL for REST source connector."; - private static final String SOURCE_URL_DISPLAY = "URL for REST source connector."; - - static final String SOURCE_DATA_CONFIG = "rest.source.data"; - private static final String SOURCE_DATA_DOC = "The data for REST source connector."; - private static final String SOURCE_DATA_DISPLAY = "Data for REST source connector."; - private static final String SOURCE_DATA_DEFAULT = null; - - static final String SOURCE_TOPIC_SELECTOR_CONFIG = "rest.source.topic.selector"; - private static final String SOURCE_TOPIC_SELECTOR_DOC = - "The topic selector class for REST source connector."; - private static final String SOURCE_TOPIC_SELECTOR_DISPLAY = - "Topic selector class for REST source connector."; - private static final Class SOURCE_TOPIC_SELECTOR_DEFAULT = - SimpleTopicSelector.class; - - static final String SOURCE_TOPIC_LIST_CONFIG = "rest.source.destination.topics"; - private static final String SOURCE_TOPIC_LIST_DOC = - "The list of destination topics for the REST source connector."; - private static final String SOURCE_TOPIC_LIST_DISPLAY = "Source destination topics"; - - static final String SOURCE_PAYLOAD_CONVERTER_CONFIG = "rest.source.payload.converter.class"; - private static final Class PAYLOAD_CONVERTER_DEFAULT = - StringPayloadConverter.class; - private static final String SOURCE_PAYLOAD_CONVERTER_DOC_CONFIG = - "Class to be used to convert messages from REST calls to SourceRecords"; - private static final String SOURCE_PAYLOAD_CONVERTER_DISPLAY_CONFIG = "Payload converter class"; - private final TopicSelector topicSelector; - private final PayloadToSourceRecordConverter payloadToSourceRecordConverter; - private final Map requestProperties; - - @SuppressWarnings("unchecked") - private RestSourceConnectorConfig(ConfigDef config, Map parsedConfig) { - super(config, parsedConfig); - try { - topicSelector = ((Class) - getClass(SOURCE_TOPIC_SELECTOR_CONFIG)).getDeclaredConstructor().newInstance(); - payloadToSourceRecordConverter = ((Class) - getClass(SOURCE_PAYLOAD_CONVERTER_CONFIG)).getDeclaredConstructor().newInstance(); - } catch (IllegalAccessException | InstantiationException - | InvocationTargetException | NoSuchMethodException e) { - throw new ConnectException("Invalid class for: " + SOURCE_PAYLOAD_CONVERTER_CONFIG, e); - } - requestProperties = getPropertiesList().stream() - .map(a -> a.split(":")) - .collect(Collectors.toMap(a -> a[0], a -> a[1])); - - } - - public RestSourceConnectorConfig(Map parsedConfig) { - this(conf(), parsedConfig); - } - - static ConfigDef conf() { - String group = "REST"; - int orderInGroup = 0; - return new ConfigDef() - .define(SOURCE_POLL_INTERVAL_CONFIG, - Type.LONG, - SOURCE_POLL_INTERVAL_DEFAULT, - Importance.LOW, - SOURCE_POLL_INTERVAL_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_POLL_INTERVAL_DISPLAY) - - .define(SOURCE_METHOD_CONFIG, - Type.STRING, - SOURCE_METHOD_DEFAULT, - new MethodValidator(), - Importance.HIGH, - SOURCE_METHOD_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_METHOD_DISPLAY, - new MethodRecommender()) - - .define(SOURCE_PROPERTIES_LIST_CONFIG, - Type.LIST, - NO_DEFAULT_VALUE, - Importance.HIGH, - SOURCE_PROPERTIES_LIST_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_PROPERTIES_LIST_DISPLAY) - - .define(SOURCE_URL_CONFIG, - Type.STRING, - NO_DEFAULT_VALUE, - Importance.HIGH, - SOURCE_URL_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_URL_DISPLAY) - - .define(SOURCE_DATA_CONFIG, - Type.STRING, - SOURCE_DATA_DEFAULT, - Importance.LOW, - SOURCE_DATA_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_DATA_DISPLAY) - - .define(SOURCE_TOPIC_LIST_CONFIG, - Type.LIST, - NO_DEFAULT_VALUE, - Importance.HIGH, - SOURCE_TOPIC_LIST_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_TOPIC_LIST_DISPLAY) - - .define(SOURCE_TOPIC_SELECTOR_CONFIG, - Type.CLASS, - SOURCE_TOPIC_SELECTOR_DEFAULT, - new TopicSelectorValidator(), - Importance.HIGH, - SOURCE_TOPIC_SELECTOR_DOC, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_TOPIC_SELECTOR_DISPLAY, - new TopicSelectorRecommender()) - - - .define(SOURCE_PAYLOAD_CONVERTER_CONFIG, - Type.CLASS, - PAYLOAD_CONVERTER_DEFAULT, - new PayloadToSourceRecordConverterValidator(), - Importance.LOW, - SOURCE_PAYLOAD_CONVERTER_DOC_CONFIG, - group, - ++orderInGroup, - ConfigDef.Width.SHORT, - SOURCE_PAYLOAD_CONVERTER_DISPLAY_CONFIG, - new PayloadToSourceRecordConverterRecommender()) - ; - } - - Long getPollInterval() { - return this.getLong(SOURCE_POLL_INTERVAL_CONFIG); - } - - String getMethod() { - return this.getString(SOURCE_METHOD_CONFIG); - } - - private List getPropertiesList() { - return this.getList(SOURCE_PROPERTIES_LIST_CONFIG); - } - - public String getUrl() { - return this.getString(SOURCE_URL_CONFIG); - } - - public List getTopics() { - return this.getList(SOURCE_TOPIC_LIST_CONFIG); - } - - public TopicSelector getTopicSelector() { - return topicSelector; - } - - public String getData() { - return this.getString(SOURCE_DATA_CONFIG); - } - - PayloadToSourceRecordConverter getPayloadToSourceRecordConverter() { - return payloadToSourceRecordConverter; - } - - Map getRequestProperties() { - return requestProperties; - } - - private static ConfigDef getConfig() { - Map everything = new HashMap<>(conf().configKeys()); - ConfigDef visible = new ConfigDef(); - for (ConfigDef.ConfigKey key : everything.values()) { - visible.define(key); - } - return visible; - } - - public static void main(String[] args) { - System.out.println(VersionUtil.getVersion()); - System.out.println(getConfig().toEnrichedRst()); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/RestSourceTask.java b/src/main/java/com/tm/kafka/connect/rest/RestSourceTask.java deleted file mode 100644 index 757a9bc6..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/RestSourceTask.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.tm.kafka.connect.rest; - -import com.tm.kafka.connect.rest.converter.PayloadToSourceRecordConverter; -import org.apache.commons.compress.utils.IOUtils; -import org.apache.kafka.connect.source.SourceRecord; -import org.apache.kafka.connect.source.SourceTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class RestSourceTask extends SourceTask { - private static Logger log = LoggerFactory.getLogger(RestSourceTask.class); - - private Long pollInterval; - private String method; - private Map requestProperties; - private String url; - private String data; - private PayloadToSourceRecordConverter converter; - - private Long lastPollTime = 0L; - - @Override - public void start(Map map) { - RestSourceConnectorConfig connectorConfig = new RestSourceConnectorConfig(map); - pollInterval = connectorConfig.getPollInterval(); - method = connectorConfig.getMethod(); - requestProperties = connectorConfig.getRequestProperties(); - url = connectorConfig.getUrl(); - data = connectorConfig.getData(); - converter = connectorConfig.getPayloadToSourceRecordConverter(); - converter.start(connectorConfig); - } - - @Override - public List poll() throws InterruptedException { - long millis = pollInterval - (System.currentTimeMillis() - lastPollTime); - if (millis > 0) { - Thread.sleep(millis); - } - try { - HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); - requestProperties.forEach(conn::setRequestProperty); - conn.setRequestMethod(method); - if (data != null) { - conn.setDoOutput(true); - OutputStream os = conn.getOutputStream(); - os.write(data.getBytes()); - os.flush(); - } - if (log.isTraceEnabled()) { - log.trace("Response code: {}, Request data: {}", conn.getResponseCode(), data); - } - return converter.convert(IOUtils.toByteArray(conn.getInputStream())); - } catch (Exception e) { - log.error("REST source connector poll() failed", e); - return Collections.emptyList(); - } finally { - lastPollTime = System.currentTimeMillis(); - } - } - - @Override - public void stop() { - log.debug("Stopping source task"); - } - - @Override - public String version() { - return VersionUtil.getVersion(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/VersionUtil.java b/src/main/java/com/tm/kafka/connect/rest/VersionUtil.java deleted file mode 100644 index 336014c9..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/VersionUtil.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tm.kafka.connect.rest; - -class VersionUtil { - static String getVersion() { - try { - return VersionUtil.class.getPackage().getImplementationVersion(); - } catch (Exception ex) { - return "0.0.0.0"; - } - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/MethodRecommender.java b/src/main/java/com/tm/kafka/connect/rest/config/MethodRecommender.java deleted file mode 100644 index 9e304415..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/MethodRecommender.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class MethodRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Arrays.asList("GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/MethodValidator.java b/src/main/java/com/tm/kafka/connect/rest/config/MethodValidator.java deleted file mode 100644 index 5ee9e437..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/MethodValidator.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.HashMap; - -public class MethodValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object provider) { - } - - @Override - public String toString() { - return new MethodRecommender().validValues("", new HashMap<>()).toString(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.java b/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.java deleted file mode 100644 index 375f61ad..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import com.tm.kafka.connect.rest.converter.BytesPayloadConverter; -import com.tm.kafka.connect.rest.converter.StringPayloadConverter; -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class PayloadToSourceRecordConverterRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Arrays.asList(StringPayloadConverter.class, BytesPayloadConverter.class); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.java b/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.java deleted file mode 100644 index c758954d..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import com.tm.kafka.connect.rest.converter.PayloadToSourceRecordConverter; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -public class PayloadToSourceRecordConverterValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object provider) { - if (provider instanceof Class - && PayloadToSourceRecordConverter.class.isAssignableFrom((Class) provider)) { - return; - } - throw new ConfigException(name, provider, "Class must extend: " - + PayloadToSourceRecordConverter.class); - } - - @Override - public String toString() { - return "Any class implementing: " + PayloadToSourceRecordConverter.class; - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.java b/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.java deleted file mode 100644 index f7362032..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import com.tm.kafka.connect.rest.selector.SimpleTopicSelector; -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class TopicSelectorRecommender implements ConfigDef.Recommender { - @Override - public List validValues(String name, Map connectorConfigs) { - return Collections.singletonList(SimpleTopicSelector.class); - } - - @Override - public boolean visible(String name, Map connectorConfigs) { - return true; - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorValidator.java b/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorValidator.java deleted file mode 100644 index 6eacafa2..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/config/TopicSelectorValidator.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tm.kafka.connect.rest.config; - -import com.tm.kafka.connect.rest.selector.TopicSelector; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -public class TopicSelectorValidator implements ConfigDef.Validator { - @Override - public void ensureValid(String name, Object topicSelector) { - if (topicSelector instanceof Class - && TopicSelector.class.isAssignableFrom((Class) topicSelector)) { - return; - } - throw new ConfigException(name, topicSelector, "Class must extend: " + TopicSelector.class); - } - - @Override - public String toString() { - return "Any class implementing: " + TopicSelector.class; - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.java deleted file mode 100644 index f680a4a5..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import com.tm.kafka.connect.rest.RestSourceConnectorConfig; -import com.tm.kafka.connect.rest.selector.TopicSelector; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.source.SourceRecord; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static java.lang.System.currentTimeMillis; - -public class BytesPayloadConverter - implements SinkRecordToPayloadConverter, PayloadToSourceRecordConverter { - - private TopicSelector topicSelector; - private String url; - - // Convert to String for outgoing REST calls - public String convert(SinkRecord record) { - return record.value().toString(); - } - - // Just bytes for incoming messages - public List convert(byte[] bytes) { - ArrayList records = new ArrayList<>(); - Map sourcePartition = Collections.singletonMap("URL", url); - Map sourceOffset = Collections.singletonMap("timestamp", currentTimeMillis()); - records.add(new SourceRecord(sourcePartition, sourceOffset, topicSelector.getTopic(bytes), - Schema.BYTES_SCHEMA, bytes)); - return records; - } - - @Override - public void start(RestSourceConnectorConfig config) { - url = config.getUrl(); - topicSelector = config.getTopicSelector(); - topicSelector.start(config); - } - - @Override - public void start(RestSinkConnectorConfig config) { - url = config.getUrl(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.java deleted file mode 100644 index be346ef5..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import org.apache.kafka.connect.sink.SinkRecord; - -import java.io.IOException; - -public class JacksonPayloadConverter - implements SinkRecordToPayloadConverter { - - private ObjectMapper mapper = new ObjectMapper(); - - // Convert to a String for outgoing REST calls - public String convert(SinkRecord record) { - try { - return mapper.writeValueAsString(record.value()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void start(RestSinkConnectorConfig config) { - - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.java deleted file mode 100644 index 244f6fe1..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.storage.ConverterType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; - -public class JsonPayloadConverter implements SinkRecordToPayloadConverter { - private Logger log = LoggerFactory.getLogger(JsonPayloadConverter.class); - private JsonConverter converter = new JsonConverter(); - - @Override - public String convert(SinkRecord record) { - String topic = record.topic(); - Object value = record.value(); - Schema schema = record.valueSchema(); - - byte[] result = converter.fromConnectData(topic, schema, value); - String resultStr = new String(result); - - if (log.isTraceEnabled()) { - log.trace(String.format("Record %s -> JSON %s", record.toString(), resultStr)); - } - return resultStr; - } - - @Override - public void start(RestSinkConnectorConfig config) { - Map map = new HashMap<>(); - map.put("schemas.enable", config.getIncludeSchema().toString()); - map.put("converter.type", ConverterType.VALUE.getName()); - - converter.configure(map); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.java deleted file mode 100644 index 4d54531d..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.tm.kafka.connect.rest.RestSourceConnectorConfig; -import org.apache.kafka.connect.source.SourceRecord; - -import java.util.List; - -public interface PayloadToSourceRecordConverter { - List convert(final byte[] bytes) throws Exception; - - void start(RestSourceConnectorConfig config); -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.java deleted file mode 100644 index 33486f5e..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import org.apache.kafka.connect.sink.SinkRecord; - -public interface SinkRecordToPayloadConverter { - String convert(final SinkRecord record) throws Exception; - - void start(RestSinkConnectorConfig config); -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/StringPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/StringPayloadConverter.java deleted file mode 100644 index bda453ac..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/StringPayloadConverter.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import com.tm.kafka.connect.rest.RestSourceConnectorConfig; -import com.tm.kafka.connect.rest.selector.TopicSelector; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.source.SourceRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static java.lang.System.currentTimeMillis; - -public class StringPayloadConverter - implements SinkRecordToPayloadConverter, PayloadToSourceRecordConverter { - private Logger log = LoggerFactory.getLogger(StringPayloadConverter.class); - - private String url; - private TopicSelector topicSelector; - - public String convert(SinkRecord record) { - if (log.isTraceEnabled()) { - log.trace("SinkRecord: {}", record.toString()); - } - - return record.value().toString(); - } - - public List convert(byte[] bytes) { - ArrayList records = new ArrayList<>(); - Map sourcePartition = Collections.singletonMap("URL", url); - Map sourceOffset = Collections.singletonMap("timestamp", currentTimeMillis()); - String topic = topicSelector.getTopic(bytes); - String value = new String(bytes); - SourceRecord sourceRecord = new SourceRecord(sourcePartition, sourceOffset, topic, - Schema.STRING_SCHEMA, value); - if (log.isTraceEnabled()) { - log.trace("SourceRecord: {}", sourceRecord); - } - records.add(sourceRecord); - return records; - } - - @Override - public void start(RestSourceConnectorConfig config) { - url = config.getUrl(); - topicSelector = config.getTopicSelector(); - topicSelector.start(config); - } - - @Override - public void start(RestSinkConnectorConfig config) { - url = config.getUrl(); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.java b/src/main/java/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.java deleted file mode 100644 index ef5cb1e3..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.tm.kafka.connect.rest.converter; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.tm.kafka.connect.rest.RestSinkConnectorConfig; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.velocity.Template; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Map; - - -public class VelocityPayloadConverter implements SinkRecordToPayloadConverter { - private Logger log = LoggerFactory.getLogger(VelocityPayloadConverter.class); - private ObjectMapper mapper = new ObjectMapper(); - - private VelocityContext globalContext; - private Template template; - - public String convert(SinkRecord record) throws IOException { - StringWriter sw = new StringWriter(); - - VelocityContext context = new VelocityContext(globalContext); - - context.put("topic", record.topic()); - context.put("partition", record.kafkaPartition()); - context.put("key", record.key()); - context.put("timestamp", record.timestamp()); - context.put("schema", record.valueSchema()); - context.put("value", mapper.readValue((String) record.value(), Map.class)); - - template.merge(context, sw); - return sw.toString(); - } - - @Override - public void start(RestSinkConnectorConfig config) { - Velocity.init(); - globalContext = new VelocityContext(); - template = Velocity.getTemplate(config.getVelocityTemplate()); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.java b/src/main/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.java deleted file mode 100644 index 90695e7f..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tm.kafka.connect.rest.selector; - -import com.tm.kafka.connect.rest.RestSourceConnectorConfig; - -public class SimpleTopicSelector implements TopicSelector { - private String topic; - - @Override - public String getTopic(Object data) { - return topic; - } - - @Override - public void start(RestSourceConnectorConfig config) { - // Always return the first topic in the list - topic = config.getTopics().get(0); - } -} diff --git a/src/main/java/com/tm/kafka/connect/rest/selector/TopicSelector.java b/src/main/java/com/tm/kafka/connect/rest/selector/TopicSelector.java deleted file mode 100644 index d2148dd2..00000000 --- a/src/main/java/com/tm/kafka/connect/rest/selector/TopicSelector.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.tm.kafka.connect.rest.selector; - -import com.tm.kafka.connect.rest.RestSourceConnectorConfig; - -public interface TopicSelector { - String getTopic(Object data); - - void start(RestSourceConnectorConfig config); -} diff --git a/src/test/java/com/tm/kafka/connect/rest/RestConnectorConfigTest.java b/src/test/java/com/tm/kafka/connect/rest/RestConnectorConfigTest.java deleted file mode 100644 index 3372b4e4..00000000 --- a/src/test/java/com/tm/kafka/connect/rest/RestConnectorConfigTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.tm.kafka.connect.rest; - -import org.junit.Test; - -public class RestConnectorConfigTest { - @Test - public void docSource() { - System.out.println(RestSourceConnectorConfig.conf().toRst()); - } - - @Test - public void docSink() { - System.out.println(RestSinkConnectorConfig.conf().toRst()); - } -} diff --git a/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.java b/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.java deleted file mode 100644 index 2738699c..00000000 --- a/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.tm.kafka.connect.rest; - -import org.junit.Test; - -public class RestSinkConnectorConfigTest { - @Test - public void doc() { - System.out.println(RestSinkConnectorConfig.conf().toRst()); - } -} diff --git a/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorTest.java b/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorTest.java deleted file mode 100644 index e89c572b..00000000 --- a/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.tm.kafka.connect.rest; - -import org.junit.Test; - -public class RestSinkConnectorTest { - @Test - public void test() { - // Congrats on a passing test! - } -} diff --git a/src/test/java/com/tm/kafka/connect/rest/RestTaskTest.java b/src/test/java/com/tm/kafka/connect/rest/RestTaskTest.java deleted file mode 100644 index 0ed7331c..00000000 --- a/src/test/java/com/tm/kafka/connect/rest/RestTaskTest.java +++ /dev/null @@ -1,263 +0,0 @@ -package com.tm.kafka.connect.rest; - -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.record.TimestampType; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTaskContext; -import org.apache.kafka.connect.source.SourceRecord; -import org.junit.Rule; -import org.junit.Test; - -import java.net.ServerSocket; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.matching; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; -import static com.github.tomakehurst.wiremock.client.WireMock.verify; -import static org.apache.kafka.connect.data.Schema.STRING_SCHEMA; -import static org.junit.Assert.assertEquals; - -public class RestTaskTest { - - private static final String CONTENT_TYPE = "Content-Type"; - private static final String ACCEPT = "Accept"; - private static final String APPLICATION_JSON = "application/json"; - - private static final String TOPIC = "rest-source-destination-topic"; - private static final String REST_SOURCE_DESTINATION_TOPIC_LIST = TOPIC; - - private static final String METHOD = "POST"; - private static final String PROPERTIES_LIST = "" + - CONTENT_TYPE + ":" + APPLICATION_JSON + ", " + - ACCEPT + ":" + APPLICATION_JSON; - private static final String TOPIC_SELECTOR = "com.tm.kafka.connect.rest.selector.SimpleTopicSelector"; - private static final String BYTES_PAYLOAD_CONVERTER = "com.tm.kafka.connect.rest.converter.BytesPayloadConverter"; - private static final String STRING_PAYLOAD_CONVERTER = "com.tm.kafka.connect.rest.converter.StringPayloadConverter"; - private static final String DATA = "{\"A\":\"B\"}"; - private static final String RESPONSE_BODY = "{\"B\":\"A\"}"; - private static final int PORT = getPort(); - private static final String PATH = "/my/resource"; - private static final String URL = "http://localhost:" + PORT + PATH; - - // Sink - private static final int PARTITION12 = 12; - private static final int PARTITION13 = 13; - private static final TopicPartition TOPIC_PARTITION = new TopicPartition(TOPIC, PARTITION12); - private static final TopicPartition TOPIC_PARTITION2 = new TopicPartition(TOPIC, PARTITION13); - - @Rule - public WireMockRule wireMockRule = new WireMockRule(PORT); - - @Test - public void restTest() throws InterruptedException { - stubFor(post(urlEqualTo(PATH)) - .withHeader(ACCEPT, equalTo(APPLICATION_JSON)) - .willReturn(aResponse() - .withStatus(200) - .withHeader(CONTENT_TYPE, APPLICATION_JSON) - .withBody(RESPONSE_BODY))); - - Map props; - props = new HashMap() {{ - put(RestSourceConnectorConfig.SOURCE_METHOD_CONFIG, METHOD); - put(RestSourceConnectorConfig.SOURCE_PROPERTIES_LIST_CONFIG, PROPERTIES_LIST); - put(RestSourceConnectorConfig.SOURCE_URL_CONFIG, URL); - put(RestSourceConnectorConfig.SOURCE_DATA_CONFIG, DATA); - put(RestSourceConnectorConfig.SOURCE_TOPIC_SELECTOR_CONFIG, TOPIC_SELECTOR); - put(RestSourceConnectorConfig.SOURCE_TOPIC_LIST_CONFIG, REST_SOURCE_DESTINATION_TOPIC_LIST); - put(RestSourceConnectorConfig.SOURCE_PAYLOAD_CONVERTER_CONFIG, STRING_PAYLOAD_CONVERTER); - }}; - - RestSourceTask sourceTask; - List messages; - - sourceTask = new RestSourceTask(); - sourceTask.initialize(() -> null); - sourceTask.start(props); - messages = sourceTask.poll(); - - assertEquals("Message count: ", 1, messages.size()); - assertEquals("Response class: ", String.class, messages.get(0).value().getClass()); - assertEquals("Response body: ", RESPONSE_BODY, messages.get(0).value()); - assertEquals("Topic: ", TOPIC, messages.get(0).topic()); - - verify(postRequestedFor(urlMatching(PATH)) - .withRequestBody(equalTo(DATA)) - .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); - - props.put(RestSourceConnectorConfig.SOURCE_PAYLOAD_CONVERTER_CONFIG, BYTES_PAYLOAD_CONVERTER); - - sourceTask = new RestSourceTask(); - sourceTask.initialize(() -> null); - sourceTask.start(props); - messages = sourceTask.poll(); - - assertEquals("Message count: ", 1, messages.size()); - assertEquals("Response class: ", byte[].class, messages.get(0).value().getClass()); - assertEquals("Response body: ", RESPONSE_BODY, new String((byte[]) messages.get(0).value())); - assertEquals("Topic: ", TOPIC, messages.get(0).topic()); - - verify(postRequestedFor(urlMatching(PATH)) - .withRequestBody(equalTo(DATA)) - .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); - - wireMockRule.resetRequests(); - - props = new HashMap() {{ - put(RestSinkConnectorConfig.SINK_METHOD_CONFIG, METHOD); - put(RestSinkConnectorConfig.SINK_URL_CONFIG, URL); - put(RestSinkConnectorConfig.SINK_PROPERTIES_LIST_CONFIG, PROPERTIES_LIST); - put(RestSinkConnectorConfig.SINK_PAYLOAD_CONVERTER_CONFIG, STRING_PAYLOAD_CONVERTER); - }}; - String key = "key1"; - - long offset = 100; - long timestamp = 200L; - - ArrayList records = new ArrayList<>(); - records.add( - new SinkRecord( - TOPIC, - PARTITION12, - STRING_SCHEMA, - key, - STRING_SCHEMA, - DATA, - offset, - timestamp, - TimestampType.CREATE_TIME - )); - - RestSinkTask sinkTask; - sinkTask = new RestSinkTask(); - Set assignment = new HashSet<>(); - assignment.add(TOPIC_PARTITION); - assignment.add(TOPIC_PARTITION2); - MockSinkTaskContext context = new MockSinkTaskContext(assignment); - sinkTask.initialize(context); - sinkTask.start(props); - sinkTask.put(records); - - verify(postRequestedFor(urlMatching(PATH)) - .withRequestBody(equalTo(DATA)) - .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); - - wireMockRule.resetAll(); - - props.put(RestSinkConnectorConfig.SINK_METHOD_CONFIG, "GET"); - props.put(RestSinkConnectorConfig.SINK_URL_CONFIG, URL + "?"); - - stubFor(get(urlMatching(PATH + ".*")) - .withHeader(ACCEPT, equalTo(APPLICATION_JSON)) - .willReturn(aResponse() - .withStatus(200) - .withHeader(CONTENT_TYPE, APPLICATION_JSON) - .withBody(RESPONSE_BODY))); - - sinkTask = new RestSinkTask(); - sinkTask.initialize(context); - sinkTask.start(props); - - records.clear(); - Object value = "{\"id\":1, \"content\":\"Joe\"}"; - records.add( - new SinkRecord( - TOPIC, - PARTITION12, - STRING_SCHEMA, - key, - STRING_SCHEMA, - value, - offset, - timestamp, - TimestampType.CREATE_TIME - )); - - sinkTask.put(records); - - verify(getRequestedFor(urlMatching(PATH + "\\?.*")) - .withHeader(CONTENT_TYPE, matching(APPLICATION_JSON))); - } - - private static int getPort() { - try { - ServerSocket s = new ServerSocket(0); - int localPort = s.getLocalPort(); - s.close(); - return localPort; - } catch (Exception e) { - throw new RuntimeException("Failed to get a free PORT", e); - } - } - - protected static class MockSinkTaskContext implements SinkTaskContext { - - private final Map offsets; - private long timeoutMs; - private Set assignment; - - MockSinkTaskContext(Set assignment) { - this.offsets = new HashMap<>(); - this.timeoutMs = -1L; - this.assignment = assignment; - } - - @Override - public void offset(Map offsets) { - this.offsets.putAll(offsets); - } - - @Override - public void offset(TopicPartition tp, long offset) { - offsets.put(tp, offset); - } - - public Map offsets() { - return offsets; - } - - @Override - public void timeout(long timeoutMs) { - this.timeoutMs = timeoutMs; - } - - public long timeout() { - return timeoutMs; - } - - @Override - public Set assignment() { - return assignment; - } - - public void setAssignment(Set nextAssignment) { - assignment = nextAssignment; - } - - @Override - public void pause(TopicPartition... partitions) { - } - - @Override - public void resume(TopicPartition... partitions) { - } - - @Override - public void requestCommit() { - } - } -} diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnector.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnector.class deleted file mode 100644 index a2daca5e..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnector.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$1.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$1.class deleted file mode 100644 index fd16f314..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$1.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodRecommender.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodRecommender.class deleted file mode 100644 index c4251209..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodValidator.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodValidator.class deleted file mode 100644 index 7d35d10b..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$MethodValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterRecommender.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterRecommender.class deleted file mode 100644 index c4191bb4..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaRecommender.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaRecommender.class deleted file mode 100644 index 52383a29..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaValidator.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaValidator.class deleted file mode 100644 index e7d0b1aa..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterSchemaValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterValidator.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterValidator.class deleted file mode 100644 index 2c43686b..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig$PayloadConverterValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig.class b/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig.class deleted file mode 100644 index 396baf3b..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkConnectorConfig.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSinkTask.class b/target/classes/com/tm/kafka/connect/rest/RestSinkTask.class deleted file mode 100644 index 656b243b..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSinkTask.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSourceConnector.class b/target/classes/com/tm/kafka/connect/rest/RestSourceConnector.class deleted file mode 100644 index 82692604..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSourceConnector.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSourceConnectorConfig.class b/target/classes/com/tm/kafka/connect/rest/RestSourceConnectorConfig.class deleted file mode 100644 index cf72e4cf..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSourceConnectorConfig.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/RestSourceTask.class b/target/classes/com/tm/kafka/connect/rest/RestSourceTask.class deleted file mode 100644 index 85cebcd6..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/RestSourceTask.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/VersionUtil.class b/target/classes/com/tm/kafka/connect/rest/VersionUtil.class deleted file mode 100644 index 78b7b051..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/VersionUtil.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/MethodRecommender.class b/target/classes/com/tm/kafka/connect/rest/config/MethodRecommender.class deleted file mode 100644 index e6eb27aa..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/MethodRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/MethodValidator.class b/target/classes/com/tm/kafka/connect/rest/config/MethodValidator.class deleted file mode 100644 index d5c485f1..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/MethodValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.class b/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.class deleted file mode 100644 index 6c32f1d0..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.class b/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.class deleted file mode 100644 index 4904b3e8..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/PayloadToSourceRecordConverterValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.class b/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.class deleted file mode 100644 index e7333116..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorRecommender.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorValidator.class b/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorValidator.class deleted file mode 100644 index cb8ce56a..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/config/TopicSelectorValidator.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.class deleted file mode 100644 index c2e1894b..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/BytesPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.class deleted file mode 100644 index 2a46002d..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/JacksonPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.class deleted file mode 100644 index 89268b49..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/JsonPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.class deleted file mode 100644 index 8d808ab2..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/PayloadToSourceRecordConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.class deleted file mode 100644 index 6ae83982..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/SinkRecordToPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/StringPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/StringPayloadConverter.class deleted file mode 100644 index 779898d3..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/StringPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.class b/target/classes/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.class deleted file mode 100644 index 143c60b3..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/converter/VelocityPayloadConverter.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.class b/target/classes/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.class deleted file mode 100644 index 0bac6401..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.class and /dev/null differ diff --git a/target/classes/com/tm/kafka/connect/rest/selector/TopicSelector.class b/target/classes/com/tm/kafka/connect/rest/selector/TopicSelector.class deleted file mode 100644 index 9d99c8f0..00000000 Binary files a/target/classes/com/tm/kafka/connect/rest/selector/TopicSelector.class and /dev/null differ diff --git a/target/kafka-connect-rest-1.0-SNAPSHOT-shaded.jar b/target/kafka-connect-rest-1.0-SNAPSHOT-shaded.jar deleted file mode 100644 index 9f523195..00000000 Binary files a/target/kafka-connect-rest-1.0-SNAPSHOT-shaded.jar and /dev/null differ diff --git a/target/kafka-connect-rest-1.0-SNAPSHOT.jar b/target/kafka-connect-rest-1.0-SNAPSHOT.jar deleted file mode 100644 index ceeff395..00000000 Binary files a/target/kafka-connect-rest-1.0-SNAPSHOT.jar and /dev/null differ diff --git a/target/maven-archiver/pom.properties b/target/maven-archiver/pom.properties deleted file mode 100644 index fcdd508a..00000000 --- a/target/maven-archiver/pom.properties +++ /dev/null @@ -1,4 +0,0 @@ -#Created by Apache Maven 3.5.3 -groupId=com.tm.kafka -artifactId=kafka-connect-rest -version=1.0-SNAPSHOT diff --git a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestConnectorConfigTest.xml b/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestConnectorConfigTest.xml deleted file mode 100644 index 63525c71..00000000 --- a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestConnectorConfigTest.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.xml b/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.xml deleted file mode 100644 index fbb51ba2..00000000 --- a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorTest.xml b/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorTest.xml deleted file mode 100644 index e10025ee..00000000 --- a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestSinkConnectorTest.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestTaskTest.xml b/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestTaskTest.xml deleted file mode 100644 index 8e780808..00000000 --- a/target/surefire-reports/TEST-com.tm.kafka.connect.rest.RestTaskTest.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/target/surefire-reports/com.tm.kafka.connect.rest.RestConnectorConfigTest.txt b/target/surefire-reports/com.tm.kafka.connect.rest.RestConnectorConfigTest.txt deleted file mode 100644 index 1c3375da..00000000 --- a/target/surefire-reports/com.tm.kafka.connect.rest.RestConnectorConfigTest.txt +++ /dev/null @@ -1,4 +0,0 @@ -------------------------------------------------------------------------------- -Test set: com.tm.kafka.connect.rest.RestConnectorConfigTest -------------------------------------------------------------------------------- -Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.067 sec diff --git a/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.txt b/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.txt deleted file mode 100644 index 4e99a45b..00000000 --- a/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorConfigTest.txt +++ /dev/null @@ -1,4 +0,0 @@ -------------------------------------------------------------------------------- -Test set: com.tm.kafka.connect.rest.RestSinkConnectorConfigTest -------------------------------------------------------------------------------- -Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.001 sec diff --git a/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorTest.txt b/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorTest.txt deleted file mode 100644 index e23014f5..00000000 --- a/target/surefire-reports/com.tm.kafka.connect.rest.RestSinkConnectorTest.txt +++ /dev/null @@ -1,4 +0,0 @@ -------------------------------------------------------------------------------- -Test set: com.tm.kafka.connect.rest.RestSinkConnectorTest -------------------------------------------------------------------------------- -Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0 sec diff --git a/target/surefire-reports/com.tm.kafka.connect.rest.RestTaskTest.txt b/target/surefire-reports/com.tm.kafka.connect.rest.RestTaskTest.txt deleted file mode 100644 index 9515439c..00000000 --- a/target/surefire-reports/com.tm.kafka.connect.rest.RestTaskTest.txt +++ /dev/null @@ -1,4 +0,0 @@ -------------------------------------------------------------------------------- -Test set: com.tm.kafka.connect.rest.RestTaskTest -------------------------------------------------------------------------------- -Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.555 sec diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestConnectorConfigTest.class b/target/test-classes/com/tm/kafka/connect/rest/RestConnectorConfigTest.class deleted file mode 100644 index fe8ec2e6..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestConnectorConfigTest.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.class b/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.class deleted file mode 100644 index 0c655725..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorTest.class b/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorTest.class deleted file mode 100644 index e24e915f..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestSinkConnectorTest.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$1.class b/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$1.class deleted file mode 100644 index 3e22e7d0..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$1.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$2.class b/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$2.class deleted file mode 100644 index bc6d5bec..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$2.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$MockSinkTaskContext.class b/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$MockSinkTaskContext.class deleted file mode 100644 index 72c2b139..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest$MockSinkTaskContext.class and /dev/null differ diff --git a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest.class b/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest.class deleted file mode 100644 index a56ac9c9..00000000 Binary files a/target/test-classes/com/tm/kafka/connect/rest/RestTaskTest.class and /dev/null differ diff --git a/target/test-classes/logback.xml b/target/test-classes/logback.xml deleted file mode 100644 index 1e3b4bb5..00000000 --- a/target/test-classes/logback.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - -