Merge pull request #404 from DLTcollab/develop

Pre-release v0.8.0

.bazelrc

@@ -1,26 +1,53 @@
-test --copt='-ggdb3'
+build --copt -W
+build --copt -Wall
+build --copt -Wextra
+build --output_filter='^//((?!external:).)*$'
+
 coverage -s
 coverage --experimental_cc_coverage
 coverage --combined_report=lcov
 coverage --coverage_report_generator=@bazel_tools//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:Main
 coverage --instrumentation_filter="-/tests[/:]"
 
-# Address Sanitizer:--config asan
+
+test --copt='-ggdb3'
+
+# enforce using python2
+# see also: https://github.com/bazelbuild/rules_docker#known-issues
+build --host_force_python=PY2
+test --host_force_python=PY2
+run --host_force_python=PY2
+
+# --config asan: Address sanitizer
 build:asan --strip=never
+build:asan --copt -DADDRESS_SANITIZER
 build:asan --copt -fsanitize=address
 build:asan --copt -fno-omit-frame-pointer
 build:asan --linkopt -fsanitize=address
 
-# Thread Sanitizer: --config tsan
+# --config tsan: ThreadSanitizer
 build:tsan --strip=never
+build:tsan --copt -DTHREAD_SANITIZER
+build:tsan --copt -DDYNAMIC_ANNOTATIONS_ENABLED=1
+build:tsan --copt -DDYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1
 build:tsan --copt -fsanitize=thread
 build:tsan --copt -fno-omit-frame-pointer
 build:tsan --linkopt -fsanitize=thread
+build:tsan --linkopt -ltsan
+
+# --config msan: Memory sanitizer
+build:msan --strip=never
+build:msan --copt -DADDRESS_SANITIZER
+build:msan --copt -fsanitize=memory
+build:msan --copt -fno-omit-frame-pointer
+build:msan --linkopt -fsanitize=memory
 
-# Undefined Behavior Sanitizer: --config ubsan
+# --config ubsan: Undefined Behavior Sanitizer
 build:ubsan --strip=never
 build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -fno-omit-frame-pointer
 build:ubsan --linkopt -fsanitize=undefined
+build:ubsan --linkopt -lubsan
 
+import %workspace%/hooks/toolchains.rc
 try-import %workspace%/user.bazelrc

.clang-format

@@ -43,7 +43,7 @@ BreakConstructorInitializersBeforeComma: false
 BreakConstructorInitializers: BeforeColon
 BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: true
-ColumnLimit:     80
+ColumnLimit:     120
 CommentPragmas:  '^ IWYU pragma:'
 CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "third_party/hiredis"]
 	path = third_party/hiredis
 	url = https://github.com/redis/hiredis.git
+[submodule "third_party/mosquitto"]
+	path = third_party/mosquitto
+	url = https://github.com/eclipse/mosquitto.git

Doxyfile

@@ -8,7 +8,8 @@ INPUT                  = . \
                         request \
                         response \
                         serializer \
-                        utils
+                        utils \ 
+                        connectivity/mqtt
 FILE_PATTERNS          = *.h \
                         *.md
 EXAMPLE_PATH           = tests

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (C) 2018-2019 BiiLabs Co., Ltd. and Contributors
+Copyright (C) 2018-2020 BiiLabs Co., Ltd. and Contributors
 All Rights Reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

MQTT_server.md

@@ -0,0 +1,32 @@
+# Introduction
+The structure of this information deisseminating network is based on MQTT protocol which consists of three components. `Connectivity Endpoint`, `tangle-accelerator` and `Parser`
+
+### 1. Connectivity Endpoint
+This is a client implemented with any possible lightweight computing node (such as Raspberry Pi) and a communication module (i.e. NB-IoT module). There are some modems allowing users to send MQTT message with AT-command-like command, so for these kinds of modems, they don't need to run a `mosquitto`-dependent programs on the lightweight computing node.
+For the `Connectivity Endpoint` which uses a modem provides AT-command-like commands, few things they need to do are choosing the right topic (we use different topics to simulate RESTful methodology) and serializing the data/message into the demanded format which contains `Device ID` for TA requests. After the message is serialized into the demanded format, we can send this message with modem provided command as simple as we send a http request with Python.
+
+### 2. tangle-accelerator
+TA (tangle-accelerator) plays a role of server which receives requests from communication module and processes the requests. However, under the structure of this MQTT information deisseminating network, both `tangle-accelerator` and `Connectivity Endpoint` are MQTT client. We must take care that `tangle-accelerator` is not a broker under this topology of MQTT information disseminating network.
+`tangle-accelerator` runs as a MQTT subscriber and publisher at the same time which listen to the requests on several different topics and respond the requests according to respective request result. We treat each topic as different URL path of http protocol does.
+
+### 3. Parser
+The `Parser` plays a role of both subscriber and publisher simultaneously; thus, we will implement this `Parser`, which is a duplex client by modifying `mosquitto`'s client source code.
+Sometimes, it might necessitate parsing MQTT messages, since the messages which are sent from multifarious modems may vary from one to another. In order to support a wide range of modems from different manufactures, we can use a duplex client to revise the messages which are sent from modems into the regulated format, then send the message in regulated format to a specific topic which contains only requests follow TA's request format.
+
+## Communication structure
+`<root>` is the host operator defined root path. It can be used if we want to choose a certain host of a cluster.
+
+#### 1. Communication Endpoint sends message
+Communication endpoint would send message on topic `<root>/<API>/raw/<parser type ID>`
+On this topic, the messages are in raw types whose formats depend on the modems users chose.
+And the sub-topic, `<parser type ID>`, aims to pass the message to a appropriate parser (the work that parser should 
+do is corresponding to multifarious manufacturers or even different modems).
+
+The message should contain the `Device ID` of the `Communication Endpoint`, since the `Device ID` will be used in returning data for `tangle-accelerator`. Once the message is published, Communication Endpoint will start to listen the topic, `<root>/<API>/<Device ID>`. The resonse will be published into this topic
+#### 2. Parser parses raw message
+Parser parses messages came from topic `<root>/<API>/raw/<parser type ID>` into regulted format of TA.
+#### 3. Parser sends neat messages
+Parser sends parsed, neat, regulated format messages to topic `<root>/<API>`, and the messages contain `Device ID` of source devices as well.
+#### 3. TA receives the message
+TA receive the message from topic `<root>/<API>`, and then it starts to process the requests.
+The responses will be sent onto topic `<root>/<API>/<Device ID>`.

Makefile

@@ -1,12 +1,12 @@
 DCURL_DIR := third_party/dcurl
 DCURL_LIB := $(DCURL_DIR)/build/libdcurl.so
-HIREDIS_DIR := third_party/hiredis
-HIREDIS_LIB := $(HIREDIS_DIR)/build/libdhiredis.a
-DEPS += $(DCURL_LIB) $(HIREDIS_LIB)
+MOSQITTO_DIR := third_party/mosquitto
+MOSQITTO_LIB := $(MOSQITTO_DIR)/lib/libmosquitto.so.1
+DEPS += $(DCURL_LIB)
 
 all: $(DEPS)
 
-.PHONY: $(DCURL_LIB)
+.PHONY: $(DCURL_LIB) $(MOSQITTO_LIB)
 
 $(DCURL_LIB): $(DCURL_DIR)
 	git submodule update --init $^
@@ -15,14 +15,16 @@ $(DCURL_LIB): $(DCURL_DIR)
 	$(info Modify $^/build/local.mk for your environments.)
 	$(MAKE) -C $^ all
 
-$(HIREDIS_LIB): $(HIREDIS_DIR)
+MQTT: $(DCURL_LIB) $(MOSQITTO_LIB)
+$(MOSQITTO_LIB): $(MOSQITTO_DIR)
 	git submodule update --init $^
-	$(MAKE) -C $^ static
+	@echo
+	$(MAKE) -C $^ WITH_DOCS=no
 
 clean:
 	$(MAKE) -C $(DCURL_DIR) clean
-	$(MAKE) -C $(HIREDIS_DIR) clean
+	$(MAKE) -C $(MOSQITTO_LIB) clean
 
 distclean: clean
-	$(RM) -r $(DCURL_DIR) $(HIREDIS_DIR)
-	git checkout $(DCURL_DIR) $(HIREDIS_DIR)
+	$(RM) -r $(DCURL_DIR)
+	git checkout $(DCURL_DIR)

README.md

@@ -1,16 +1,21 @@
 # Tangle-accelerator
 
-[![Build Status](https://badge.buildkite.com/0deb4c46f2f69363e4d326014843b92853733f243f379c70b5.svg)](https://buildkite.com/dltcollab/tangle-accelerator-test) [![Gitter](https://img.shields.io/gitter/room/DLTcollab/tangle-accelerator.svg)](https://gitter.im/DLTcollab/tangle-accelerator) [![GitHub release](https://img.shields.io/github/release-pre/DLTcollab/tangle-accelerator.svg)](https://github.com/DLTcollab/tangle-accelerator/releases)
+[![Build Status](https://badge.buildkite.com/0deb4c46f2f69363e4d326014843b92853733f243f379c70b5.svg)](https://buildkite.com/dltcollab/tangle-accelerator-test)  [![GitHub release](https://img.shields.io/github/release-pre/DLTcollab/tangle-accelerator.svg)](https://github.com/DLTcollab/tangle-accelerator/releases)
 
 `Tangle-accelerator` is a caching proxy server for [IOTA](https://www.iota.org/), which
 can cache API requests and rewrite their responses as needed to be routed through full
-nodes. Thus, one instance of `Tangle-accelerator` can serve thousands of Tangle requests
-at once without accessing remote full nodes frequently.
+nodes. In other words, one instance of `Tangle-accelerator` can serve thousands of IOTA
+requests at once without accessing remote full nodes frequently, that improves the
+scalability and usability of [Tangle network](https://www.iota.org/research/meet-the-tangle).
+
+Being at the edge as a key-value store, an edge-caching node powered by `Tangle-accelerator`
+does not have to communicate to typical [IOTA](https://www.iota.org/) full nodes for every API
+calls. Instead, the cached transaction data being sought is available as needed.
 
 As an intermediate server accelerateing interactions with the Tangle, it faciliates
 [dcurl](https://github.com/DLTcollab/dcurl) to perform hardware-accelerated PoW operations
 on edge devices. In the meanwhile, `Tangle-accelerator` provides shortcuts for certain
-use scenarios such as MAM and [TangleID](https://github.com/TangleID).
+use scenarios such as MAM and [TangleID](https://tangleid.github.io/).
 
 At the moment, it is not feasible to host fully-functioned full nodes on Raspberry Pi class
 Arm devices, but Raspberry Pi 3 is known to be capable to execute `Tangle-accelerator`
@@ -43,6 +48,28 @@ both footprint and startup time are behaved pretty well.
 
 ```
 
+### Transaction reattachment
+
+`Tangle-accelerator` helps to reattach pending transactions were attached from `Tangle-accelerator`.
+Reattachment increases chances of confirmation and prevents messages being pruned when full nodes perform snapshot.
+Clients should provide a uniqle ID as the identifier to each message and it's corresponding transaction hash since a new transaction hash will be generated after reattachement. 
+
+`Tangle-accelerator` uses ScyllaDB to store each transaction's ID, hash and status(Pending or confirmed). `Tangle-accelerator` will periodically check the status of pending transactions and reattach transactions which have been pended too long. Confirmed transactions will be stored into permanodes.
+
+Clients can find the transaction alone with wanted message by using the ID to query.
+
+## Connectivity
+
+`Tangle-accelerator`, at this moment, supports the following TCP/IP derived protocols:
+* `HTTP`
+* `MQTT` 
+
+### HTTP
+`HTTP` can be used in the normal internet service. User can use RESTful APIs to interact with `tangle-accelerator`.
+
+### MQTT
+`MQTT` is a lightweight communication protocol which can be used in the IoT scenarios. `Tangle-accelerator`'s support to `MQTT` allows embedded devices to write data on IOTA internet with relative low quality hardware devices. We hope this will speed up DLT into our daily life.
+
 ## Documentation
 
 This page contains basic instructions for setting up tangle-accelerator, You can generate full documentation and API reference via Doxygen. The documentation is under `docs/` after generated:
@@ -71,7 +98,7 @@ Before running tangle-accelerator, please edit binding address/port of accelerat
 $ make && bazel run //accelerator
 ```
 
-### Build from docker
+### Optional: Build Docker Images
 
 If you prefer building a docker image, tangle-accelerator also provides build rules for it. Note that you still have to edit configurations in `accelerator/config.h`.
 
@@ -82,7 +109,73 @@ $ make && bazel run //accelerator:ta_image
 There's also an easier option to pull image from docker hub then simply run with default configs. Please do remember a redis-server is still required in this way.
 
 ```
-$ docker run -d --net=host --name tangle-accelerator wusyong/tangel-accelerator:latest
+$ docker run -d --net=host --name tangle-accelerator dltcollab/tangle-accelerator
+```
+
+### Optional: Build and Push Docker Image to Docker Hub
+
+Before pushing the docker image to Docker Hub, you need to log in the docker registry:
+
+```
+$ docker login
+```
+
+Then you could push the docker image with the following command:
+
+```
+$ make && bazel run //accelerator:push_docker
+```
+
+If you get the following error message:
+
+```
+SyntaxError: invalid syntax
+----------------
+Note: The failure of target @containerregistry//:digester (with exit code 1) may have been caused by the fact that it is running under Python 3 instead of Python 2. Examine the error to determine if that appears to be the problem. Since this target is built in the host configuration, the only way to change its version is to set --host_force_python=PY2, which affects the entire build.
+
+If this error started occurring in Bazel 0.27 and later, it may be because the Python toolchain now enforces that targets analyzed as PY2 and PY3 run under a Python 2 and Python 3 interpreter, respectively. See https://github.com/bazelbuild/bazel/issues/7899 for more information.
+------------
+```
+
+Use the `--host_force_python=PY2` parameter to force the Bazel to use the Python2 in entire build.
+
+```
+$ make && bazel run //accelerator:push_docker --host_force_python=PY2
+```
+
+### Optional: Enable MQTT connectivity
+MQTT connectivity is an optional feature allowing IoT endpoint devices to collaborate with `Tangle-Accelerator`.
+
+```
+make MQTT && bazel run //accelerator:accelerator_mqtt
+```
+
+Note you may need to set up the `MQTT_HOST` and `TOPIC_ROOT` in `config.h` to connect to a MQTT broker.
+For more information for MQTT connectivity of `tangle-accelerator`, you could read `connectivity/mqtt/usage.md`.
+
+### Optional: Enable external database for transaction reattachment
+Transaction reattachment is an optional feature.
+
+You can enable it in the build time by adding option : `--define db=enable`
+
+Transaction reattachment relies on ScyllDB, you need to install the dependency by following commands.
+
+For Ubuntu Linux 16.04/x86_64:
+
+```
+wget https://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.14.1/cassandra-cpp-driver_2.14.1-1_amd64.deb
+wget https://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.14.1/cassandra-cpp-driver-dev_2.14.1-1_amd64.deb
+sudo dpkg -i cassandra-cpp-driver_2.14.1-1_amd64.deb
+sudo dpkg -i cassandra-cpp-driver-dev_2.14.1-1_amd64.deb
+```
+
+For Ubuntu Linux 18.04/x86_64:
+
+```
+wget https://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.14.1/cassandra-cpp-driver_2.14.1-1_amd64.deb
+wget https://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.14.1/cassandra-cpp-driver-dev_2.14.1-1_amd64.deb
+sudo dpkg -i cassandra-cpp-driver_2.14.1-1_amd64.deb
+sudo dpkg -i cassandra-cpp-driver-dev_2.14.1-1_amd64.deb
 ```
 
 ## Developing
@@ -111,14 +204,35 @@ Buildifier can be installed with `bazel` or `go`
 2. change directory to `buildtools`
 3. build it with bazel command, `$ bazel build //buildifier`
    The executable file will be located under `path/to/buildtools/bazel-bin`
-4. make a soft link
+4. make a soft link or move the executable file under `/usr/bin` 
 
 ### clang-format
 clang-format can be installed by command:
 - Debian/Ubuntu based systems: `$ sudo apt-get install clang-format`
 - macOS: `$ brew install clang-format`
 
 
+## Usage
+`Tangle-accelerator` currently supports two categories of APIs
+* direct API: check [wiki page](https://github.com/DLTcollab/tangle-accelerator/wiki) for details.
+* Proxy API to IRI core functionalities
+
+### IRI Porxy API
+`tangle-accelerator` allows the use of IRI core APIs. The calling process does not have to be aware of the destination machine running IRI. With the exactly same format of IRI API, `tangle-accelerator` would help users forward the request to IRI and forward the response back to users.
+We support two way to forward Proxy APIs to IRI:
+1. Bypass Proxy APIs directly to IRI.
+2. Process the Proxy APIs, then transmit them to IRI.
+
+The user can choose which way they want with CLI argument `--proxy_passthrough`.
+All the Proxy APIs are supported with the first way.
+However, the second way currently only supports the followings Proxy APIs:
+* checkConsistency
+* findTransactions
+* getBalances
+* getInclusionStates
+* getNodeInfo
+* getTrytes
+
 ## Licensing
 `Tangle-accelerator` is freely redistributable under the MIT License. Use of this source
 code is governed by a MIT-style license that can be found in the `LICENSE` file.

WORKSPACE

@@ -1,15 +1,22 @@
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+load("//third_party:third_party.bzl", "third_party_deps")
 
 git_repository(
     name = "rules_iota",
-    commit = "1cb59eea62fd1d071de213a9aa46e61e8273472d",
+    commit = "73f598ad1ce3ba79ff22d747f723d6d5cbf351e1",
     remote = "https://github.com/iotaledger/rules_iota.git",
 )
 
+git_repository(
+    name = "iota_toolchains",
+    commit = "0f93f76fdff9f091dd3a99aa3f19b5d3f4f7f7fc",
+    remote = "https://github.com/iotaledger/toolchains.git",
+)
+
 git_repository(
     name = "entangled",
-    commit = "4960865730640d23e75ffbce84d3f74264cfcd28",
     remote = "https://github.com/iotaledger/entangled.git",
+    tag = "cclient-v1.0.0-beta",
 )
 
 git_repository(
@@ -21,12 +28,26 @@ git_repository(
 git_repository(
     name = "io_bazel_rules_docker",
     remote = "https://github.com/bazelbuild/rules_docker.git",
-    tag = "v0.6.0",
+    tag = "v0.7.0",
 )
 
 load("@rules_iota//:defs.bzl", "iota_deps")
+load("@io_bazel_rules_docker//repositories:repositories.bzl", container_repositories = "repositories")
+
+container_repositories()
+
 load("@io_bazel_rules_docker//cc:image.bzl", _cc_image_repos = "repositories")
 
 iota_deps()
 
+third_party_deps()
+
 _cc_image_repos()
+
+load("@iota_toolchains//:toolchains.bzl", "setup_initial_deps")
+
+setup_initial_deps()
+
+load("@iota_toolchains//:defs.bzl", "setup_toolchains_repositories")
+
+setup_toolchains_repositories()