Merge branch 'develop' into 'master'

Develop

See merge request in3/c/in3-core!310

.gitignore

@@ -33,8 +33,6 @@ bin/
 *.class
 target/
 .cproject
-rust/in3-sys/in3-core/*
-rust/Cargo.lock
 .overcommit.yml
 c/src/third-party/hidapi/config*
 c/src/third-party/hidapi/autom4te.cache/*
@@ -49,5 +47,3 @@ c/src/third-party/hidapi/missing
 c/src/third-party/hidapi/depcomp
 c/src/third-party/hidapi/compile
 c/src/third-party/hidapi/aclocal.m4
-
-rust/in3-sys/pre_generated/in3.rs

.vscode/launch.json

@@ -29,11 +29,15 @@
             "name": "in3 cmd",
             "program": "${workspaceFolder}/build/bin/in3",
             "cwd": "${workspaceFolder}",
+            "externalConsole": true,
             "args": [
-                "-c",
-                "btc",
-                "getblock",
-                "000000000000000000103b2395f6cd94221b10d02eb9be5850303c0534307220",
+                "-ccache",
+                "-a",
+                "3",
+                "call",
+                "totalNodes():uint",
+                "-to",
+                "0x6c095a05764a23156efd9d603eada144a9b1af33",
                 "-debug"
             ]
         },
@@ -60,7 +64,8 @@
             "name": "run test",
             "program": "${workspaceFolder}/build/test/runner",
             "args": [
-                "../c/test/testdata/requests/in3_invalid_block.json",
+                "../c/test/testdata/requests/in3_total_nodes.json",
+                "-d",
                 "-t",
                 "1"
             ],
@@ -77,7 +82,7 @@
             "type": "lldb",
             "request": "launch",
             "name": "run rpcapi test",
-            "program": "${workspaceFolder}/build/test/test_rpc_api",
+            "program": "${workspaceFolder}/build/test/test_request",
             "cwd": "${workspaceFolder}/build"
         },
         {

c/ci-deploy.yml

@@ -97,7 +97,7 @@ release_mac_and_wasm:
     - cp -r ../wasm_build/module wasm/release-wasm
     - cp -r ../asmjs_build/module wasm/release-asmjs
     - cp ../java_build/lib/in3.jar lib/in3.jar
-    - cp ../python_multilib/in3/libin3/shared/* python/in3/libin3/shared/
+    - cp -r ../python_multilib/in3/libin3/shared python/in3/libin3/
     - cd ..
     - tar -zcv  --exclude=*cmake* -f in3_${CI_COMMIT_TAG}_mac.tar.gz in3-mac-wasm/
     - IPFS_RESPONSE=$(curl -X POST https://api.pinata.cloud/pinning/pinFileToIPFS -H 'Content-Type:multipart/form-data' -H 'pinata_api_key:'"$PINATA_API_KEY" -H 'pinata_secret_api_key:'"$PINATA_SECRET_API_KEY" -F file=@in3_${CI_COMMIT_TAG}_mac.tar.gz -F 'pinataMetadata={"name":"in3_'${CI_COMMIT_TAG}'_mac.tar.gz","keyValues":{"version":"${CI_COMMIT_TAG}"}}' -F 'pinataOptions={"cidVersion":0}')
@@ -137,7 +137,7 @@ release_x64:
     - cp -r ../wasm_build/module wasm/release-wasm
     - cp -r ../asmjs_build/module wasm/release-asmjs
     - cp ../java_build/lib/in3.jar lib/in3.jar
-    - cp ../python_multilib/in3/libin3/shared/* python/in3/libin3/shared/
+    - cp -r ../python_multilib/in3/libin3/shared python/in3/libin3/
     - cd ..
     - tar -zcv  --exclude=*cmake* -f in3_${CI_COMMIT_TAG}_x64.tar.gz in3-x64/
     - IPFS_RESPONSE=$(curl -X POST https://api.pinata.cloud/pinning/pinFileToIPFS -H 'Content-Type:multipart/form-data' -H 'pinata_api_key:'"$PINATA_API_KEY" -H 'pinata_secret_api_key:'"$PINATA_SECRET_API_KEY" -F file=@in3_${CI_COMMIT_TAG}_x64.tar.gz -F 'pinataMetadata={"name":"in3_'${CI_COMMIT_TAG}'_mac.tar.gz","keyValues":{"version":"${CI_COMMIT_TAG}"}}' -F 'pinataOptions={"cidVersion":0}')
@@ -174,7 +174,7 @@ release_x86:
     - cp -r ../wasm_build/module wasm/release-wasm
     - cp -r ../asmjs_build/module wasm/release-asmjs
     - cp ../java_build/lib/in3.jar lib/in3.jar
-    - cp ../python_multilib/in3/libin3/shared/* python/in3/libin3/shared/
+    - cp -r ../python_multilib/in3/libin3/shared python/in3/libin3/
     - cd ..
     - tar -zcv  --exclude=*cmake* -f in3_${CI_COMMIT_TAG}_x86.tar.gz in3-x86/
     - IPFS_RESPONSE=$(curl -X POST https://api.pinata.cloud/pinning/pinFileToIPFS -H 'Content-Type:multipart/form-data' -H 'pinata_api_key:'"$PINATA_API_KEY" -H 'pinata_secret_api_key:'"$PINATA_SECRET_API_KEY" -F file=@in3_${CI_COMMIT_TAG}_x86.tar.gz -F 'pinataMetadata={"name":"in3_'${CI_COMMIT_TAG}'_x86.tar.gz","keyValues":{"version":"${CI_COMMIT_TAG}"}}' -F 'pinataOptions={"cidVersion":0}')
@@ -211,7 +211,7 @@ release_arm7:
     - cp -r ../wasm_build/module wasm/release-wasm
     - cp -r ../asmjs_build/module wasm/release-asmjs
     - cp ../java_build/lib/in3.jar lib/in3.jar
-    - cp ../python_multilib/in3/libin3/shared/* python/in3/libin3/shared/
+    - cp -r ../python_multilib/in3/libin3/shared python/in3/libin3/
     - cd ..
     - tar -zcv  --exclude=*cmake* -f in3_${CI_COMMIT_TAG}_arm7.tar.gz in3-arm7/
     - IPFS_RESPONSE=$(curl -X POST https://api.pinata.cloud/pinning/pinFileToIPFS -H 'Content-Type:multipart/form-data' -H 'pinata_api_key:'"$PINATA_API_KEY" -H 'pinata_secret_api_key:'"$PINATA_SECRET_API_KEY" -F file=@in3_${CI_COMMIT_TAG}_arm7.tar.gz -F 'pinataMetadata={"name":"in3_'${CI_COMMIT_TAG}'_arm7.tar.gz","keyValues":{"version":"${CI_COMMIT_TAG}"}}' -F 'pinataOptions={"cidVersion":0}')
@@ -252,7 +252,7 @@ release_win:
     - cp -r ../wasm_build/module wasm/release-wasm
     - cp -r ../asmjs_build/module wasm/release-asmjs
     - cp ../java_build/lib/in3.jar lib/in3.jar
-    - cp ../python_multilib/in3/libin3/shared/* python/in3/libin3/shared/
+    - cp -r ../python_multilib/in3/libin3/shared python/in3/libin3/
     - cd ..
     - tar -zcv  --exclude=*cmake* -f in3_${CI_COMMIT_TAG}_windows.tar.gz in3-win/
     - IPFS_RESPONSE=$(curl -X POST https://api.pinata.cloud/pinning/pinFileToIPFS -H 'Content-Type:multipart/form-data' -H 'pinata_api_key:'"$PINATA_API_KEY" -H 'pinata_secret_api_key:'"$PINATA_SECRET_API_KEY" -F file=@in3_${CI_COMMIT_TAG}_windows.tar.gz -F 'pinataMetadata={"name":"in3_'${CI_COMMIT_TAG}'_windows.tar.gz","keyValues":{"version":"${CI_COMMIT_TAG}"}}' -F 'pinataOptions={"cidVersion":0}')

c/ci.yml

@@ -106,7 +106,6 @@ gcc_legacy:
   image: docker.slock.it/build-images/cmake:gcc-legacy
   extends: .conanbuild
   before_script:
-    - curl -sSL https://cmake.org/files/v3.6/cmake-3.6.1-Linux-x86_64.tar.gz | tar -xzC /opt
     - export PATH="/opt/cmake-3.6.1-Linux-x86_64/bin/:$PATH"
   variables:
     CONAN_OPTS: "-DUSE_CURL=false"

c/include/in3/client.h

@@ -227,7 +227,7 @@ typedef struct in3_chain {
   chain_id_t           chain_id;        /**< chain_id, which could be a free or based on the public ethereum networkId*/
   in3_chain_type_t     type;            /**< chaintype */
   uint64_t             last_block;      /**< last blocknumber the nodeList was updated, which is used to detect changed in the nodelist*/
-  int                  nodelist_length; /**< number of nodes in the nodeList */
+  unsigned int         nodelist_length; /**< number of nodes in the nodeList */
   in3_node_t*          nodelist;        /**< array of nodes */
   in3_node_weight_t*   weights;         /**< stats and weights recorded for each node */
   bytes_t**            init_addresses;  /**< array of addresses of nodes that should always part of the nodeList */
@@ -615,8 +615,8 @@ NONULL in3_ret_t in3_cache_init(
  * My return NULL if not found.
  */
 NONULL in3_chain_t* in3_find_chain(
-    in3_t*     c /**< the incubed client */,
-    chain_id_t chain_id /**< chain_id */
+    const in3_t* c /**< the incubed client */,
+    chain_id_t   chain_id /**< chain_id */
 );
 
 /**

c/include/in3/context.h

@@ -62,11 +62,11 @@ typedef enum ctx_type {
  * This will be used when picking the nodes to send the request to. A linked list of these structs desribe the result.
  */
 typedef struct weight {
-  in3_node_t*        node;   /**< the node definition including the url */
-  in3_node_weight_t* weight; /**< the current weight and blacklisting-stats */
-  float              s;      /**< The starting value */
-  float              w;      /**< weight value */
-  struct weight*     next;   /**< next in the linkedlist or NULL if this is the last element*/
+  unsigned int   index;   /**< index of the node in the nodelist */
+  bool           blocked; /**< if true this node has  been blocked for sending wrong responses */
+  uint32_t       s;       /**< The starting value */
+  uint32_t       w;       /**< weight value */
+  struct weight* next;    /**< next in the linkedlist or NULL if this is the last element*/
 } node_match_t;
 
 /**
@@ -445,4 +445,11 @@ NONULL void in3_ctx_add_response(
     int         data_len  /**<  the length of the data or the the string (use -1 if data is a null terminated string)*/
 );
 
+NONULL static inline in3_node_t* ctx_get_node(const in3_chain_t* chain, const node_match_t* node) {
+  return node->index < chain->nodelist_length ? chain->nodelist + node->index : NULL;
+}
+NONULL static inline in3_node_weight_t* ctx_get_node_weight(const in3_chain_t* chain, const node_match_t* node) {
+  return node->index < chain->nodelist_length ? chain->weights + node->index : NULL;
+}
+
 #endif

c/include/in3/log.h

@@ -41,6 +41,7 @@ typedef enum { LOG_TRACE,
 #define in3_log_set_prefix(prefix) in3_log_set_prefix_(prefix)
 #define in3_log_enable_prefix() in3_log_enable_prefix_()
 #define in3_log_disable_prefix() in3_log_disable_prefix_()
+#define in3_log_is_prefix_enabled() in3_log_is_prefix_enabled_()
 #define in3_log(...) in3_log_(__VA_ARGS__)
 #else
 #define in3_log_trace(...)
@@ -57,6 +58,7 @@ typedef enum { LOG_TRACE,
 #define in3_log_get_level() LOG_TRACE
 #define in3_log_set_quiet(enable)
 #define in3_log_set_prefix(prefix)
+#define in3_log_is_prefix_enabled() 0
 #define in3_log_enable_prefix()
 #define in3_log_disable_prefix()
 #define in3_log(level, file, function, line, ...) \
@@ -82,6 +84,7 @@ void            in3_log_set_quiet_(int enable);
 void            in3_log_set_prefix_(const char* prefix);
 void            in3_log_enable_prefix_();
 void            in3_log_disable_prefix_();
+int             in3_log_is_prefix_enabled_();
 
 /* in3_log() function can be made thread-safe using the in3_log_set_lock() function */
 void in3_log_(in3_log_level_t level, const char* file, const char* function, int line, const char* fmt, ...);

c/include/in3/stringbuilder.h

@@ -72,5 +72,6 @@ sb_t* sb_add_bytes(sb_t* sb, const char* prefix, const bytes_t* bytes, int len,
 sb_t* sb_add_hexuint_l(sb_t* sb, uintmax_t uint, size_t l);                                          /**< add a integer value as hexcoded, 0x-prefixed string*/
 sb_t* sb_add_escaped_chars(sb_t* sb, const char* chars);                                             /**< add chars but escapes all quotes */
 sb_t* sb_add_int(sb_t* sb, uint64_t val);                                                            /**< adds a numeric value to the stringbuilder */
+char* format_json(const char* json);                                                                 /**< format a json string and returns a new string, which needs to be freed */
 
 #endif

c/src/cmd/in3/CMakeLists.txt

@@ -41,7 +41,7 @@ if (LEDGER_NANO)
     set(LIBS ${LIBS} ledger_signer)
 endif()
 
-add_executable(in3 main.c in3_storage.c)
+add_executable(in3 main.c in3_storage.c recorder.c)
 target_compile_definitions(in3 PRIVATE _XOPEN_SOURCE=600)
 
 target_link_libraries(in3 init pk_signer ${LIBS} -lm)

c/src/cmd/in3/main.c

@@ -71,6 +71,7 @@
 #include "../../verifier/eth1/nano/chainspec.h"
 #include "../../verifier/in3_init.h"
 #include "in3_storage.h"
+#include "recorder.h"
 #include <inttypes.h>
 #include <math.h>
 #include <stdint.h>
@@ -766,6 +767,10 @@ int main(int argc, char* argv[]) {
       run_test_request = 1;
     else if (strcmp(argv[i], "-thr") == 0)
       run_test_request = 2;
+    else if (strcmp(argv[i], "-fo") == 0)
+      recorder_write_start(c, argv[++i]);
+    else if (strcmp(argv[i], "-fi") == 0)
+      recorder_read_start(c, argv[++i]);
     else if (strcmp(argv[i], "-nl") == 0)
       set_nodelist(c, argv[++i], false);
     else if (strcmp(argv[i], "-bn") == 0)
@@ -954,7 +959,7 @@ int main(int argc, char* argv[]) {
     if (run_test_request == 1) more = "WEIGHT : LAST_BLOCK";
     if (run_test_request == 2) more = "WEIGHT : NAME                   VERSION : RUNNING : HEALTH : LAST_BLOCK";
     printf("   : %-45s : %7s : %5s : %5s: %s\n------------------------------------------------------------------------------------------------\n", "URL", "BL", "CNT", "AVG", more);
-    for (int i = 0; i < chain->nodelist_length; i++) {
+    for (unsigned int i = 0; i < chain->nodelist_length; i++) {
       in3_ctx_t* ctx      = NULL;
       char*      health_s = NULL;
       if (run_test_request) {
@@ -1251,7 +1256,7 @@ int main(int argc, char* argv[]) {
 
     // if the result is a string, we remove the quotes
     if (result[0] == '"' && result[strlen(result) - 1] == '"') {
-      memmove(result, result + 1, strlen(result) + 1);
+      memmove(result, result + 1, strlen(result));
       result[strlen(result) - 1] = 0;
     }