From aaaf2044f5ce55e3f1cdaf7c1e109a724c37497b Mon Sep 17 00:00:00 2001 From: Leister Francisco Alvarado Campos Date: Thu, 24 Aug 2023 10:35:30 -0600 Subject: [PATCH] feat(api-history): add hyperion to load history --- README.md | 6 + api-history/.gitignore | 3 + api-history/hyperion/Dockerfile | 6 + .../hyperion/config/chains/local.config.json | 136 ++++++++++++ api-history/hyperion/config/connections.json | 34 +++ api-history/hyperion/docker-compose.yaml | 28 +++ api-history/infra/docker-compose.yaml | 194 ++++++++++++++++++ run.sh | 10 +- 8 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 api-history/.gitignore create mode 100644 api-history/hyperion/Dockerfile create mode 100644 api-history/hyperion/config/chains/local.config.json create mode 100644 api-history/hyperion/config/connections.json create mode 100644 api-history/hyperion/docker-compose.yaml create mode 100644 api-history/infra/docker-compose.yaml diff --git a/README.md b/README.md index 700c54f..c86f836 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,12 @@ source run.sh start_network bp 5 ./scripts/contracts.sh ``` +5. Start Hyperion + +```sh +source run.sh start_api_history +``` + ## File structure ```text title="./libre-local" diff --git a/api-history/.gitignore b/api-history/.gitignore new file mode 100644 index 0000000..f6c48d5 --- /dev/null +++ b/api-history/.gitignore @@ -0,0 +1,3 @@ +.idea +ship/ +ca.crt diff --git a/api-history/hyperion/Dockerfile b/api-history/hyperion/Dockerfile new file mode 100644 index 0000000..01d55d3 --- /dev/null +++ b/api-history/hyperion/Dockerfile @@ -0,0 +1,6 @@ +FROM node:20 +RUN npm install pm2 -g +RUN git clone https://github.com/eosrio/hyperion-history-api.git +WORKDIR /hyperion-history-api +RUN git checkout main +RUN npm install diff --git a/api-history/hyperion/config/chains/local.config.json b/api-history/hyperion/config/chains/local.config.json new file mode 100644 index 0000000..1bce62c --- /dev/null +++ b/api-history/hyperion/config/chains/local.config.json @@ -0,0 +1,136 @@ +{ + "api": { + "enabled": true, + "pm2_scaling": 1, + "node_max_old_space_size": 1024, + "chain_name": "Local Chain", + "server_addr": "0.0.0.0", + "server_port": 7000, + "stream_port": 1234, + "server_name": "localhost: 7000", + "provider_name": "Example Provider", + "provider_url": "https://example.com", + "chain_api": "", + "push_api": "", + "chain_logo_url": "", + "enable_caching": true, + "cache_life": 1, + "limits": { + "get_actions": 1000, + "get_voters": 100, + "get_links": 1000, + "get_deltas": 1000, + "get_trx_actions": 200 + }, + "access_log": false, + "chain_api_error_log": false, + "custom_core_token": "", + "enable_export_action": false, + "disable_rate_limit": false, + "rate_limit_rpm": 1000, + "rate_limit_allow": [], + "disable_tx_cache": false, + "tx_cache_expiration_sec": 3600, + "v1_chain_cache": [ + { + "path": "get_block", + "ttl": 3000 + }, + { + "path": "get_info", + "ttl": 500 + } + ] + }, + "indexer": { + "enabled": true, + "node_max_old_space_size": 4096, + "start_on": 0, + "stop_on": 0, + "rewrite": false, + "purge_queues": false, + "live_reader": true, + "live_only_mode": false, + "abi_scan_mode": false, + "fetch_block": true, + "fetch_traces": true, + "disable_reading": false, + "disable_indexing": false, + "process_deltas": true, + "disable_delta_rm": true + }, + "settings": { + "preview": false, + "chain": "local", + "eosio_alias": "eosio", + "parser": "3.2", + "auto_stop": 0, + "index_version": "v1", + "debug": false, + "bp_logs": false, + "bp_monitoring": false, + "ipc_debug_rate": 60000, + "allow_custom_abi": false, + "rate_monitoring": true, + "max_ws_payload_mb": 256, + "ds_profiling": false, + "auto_mode_switch": false, + "hot_warm_policy": false, + "custom_policy": "", + "use_global_agent": false, + "index_partition_size": 10000000, + "es_replicas": 0 + }, + "blacklists": { + "actions": [], + "deltas": [] + }, + "whitelists": { + "actions": [], + "deltas": [], + "max_depth": 10, + "root_only": false + }, + "scaling": { + "readers": 1, + "ds_queues": 1, + "ds_threads": 1, + "ds_pool_size": 1, + "indexing_queues": 1, + "ad_idx_queues": 1, + "dyn_idx_queues": 1, + "max_autoscale": 4, + "batch_size": 5000, + "resume_trigger": 5000, + "auto_scale_trigger": 20000, + "block_queue_limit": 10000, + "max_queue_limit": 100000, + "routing_mode": "round_robin", + "polling_interval": 10000 + }, + "features": { + "streaming": { + "enable": true, + "traces": true, + "deltas": true + }, + "tables": { + "proposals": true, + "accounts": true, + "voters": true + }, + "index_deltas": true, + "index_transfer_memo": true, + "index_all_deltas": true, + "deferred_trx": false, + "failed_trx": false, + "resource_limits": false, + "resource_usage": false + }, + "prefetch": { + "read": 50, + "block": 100, + "index": 500 + }, + "plugins": {} +} diff --git a/api-history/hyperion/config/connections.json b/api-history/hyperion/config/connections.json new file mode 100644 index 0000000..5fdae1e --- /dev/null +++ b/api-history/hyperion/config/connections.json @@ -0,0 +1,34 @@ +{ + "amqp": { + "host": "rabbitmq:5672", + "api": "rabbitmq:15672", + "protocol": "http", + "user": "rabbitmq", + "pass": "rabbitmq", + "vhost": "hyperion", + "frameMax": "0x10000" + }, + "elasticsearch": { + "protocol": "https", + "host": "es01:9200", + "ingest_nodes": [ + "es01:9200" + ], + "user": "elastic", + "pass": "changeme123" + }, + "redis": { + "host": "redis", + "port": "6379" + }, + "chains": { + "local": { + "name": "Libre Local", + "chain_id": "f7a3fe6bf7dc0b011e25cee41ed9526b70a9702ca74599692852304cb36fdb2b", + "http": "http://genesis:8888", + "ship": "ws://genesis:8080", + "WS_ROUTER_PORT": 7001, + "WS_ROUTER_HOST": "127.0.0.1" + } + } + } \ No newline at end of file diff --git a/api-history/hyperion/docker-compose.yaml b/api-history/hyperion/docker-compose.yaml new file mode 100644 index 0000000..e54ee19 --- /dev/null +++ b/api-history/hyperion/docker-compose.yaml @@ -0,0 +1,28 @@ +version: "3.8" +services: + indexer: + build: . + # restart: on-failure + volumes: + - ./config/connections.json:/hyperion-history-api/connections.json + - ./config/chains/:/hyperion-history-api/chains/ + networks: + - libre-network + command: ./run.sh local-indexer + + api: + build: . + ports: + - "7000:7000" + depends_on: + - indexer + volumes: + - ./config/connections.json:/hyperion-history-api/connections.json + - ./config/chains/:/hyperion-history-api/chains/ + networks: + - libre-network + command: ./run.sh local-api + +networks: + libre-network: + external: true \ No newline at end of file diff --git a/api-history/infra/docker-compose.yaml b/api-history/infra/docker-compose.yaml new file mode 100644 index 0000000..5ceaf04 --- /dev/null +++ b/api-history/infra/docker-compose.yaml @@ -0,0 +1,194 @@ +version: "3.8" + +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + networks: + - libre-network + + es01: + depends_on: + setup: + condition: service_healthy + deploy: + resources: + limits: + memory: 4G + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - es_data:/usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=basic + - xpack.monitoring.collection.enabled=true + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + networks: + - libre-network + + kibana: + profiles: + - tools + deploy: + resources: + limits: + memory: 4G + depends_on: + es01: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + volumes: + - certs:/usr/share/kibana/config/certs + - kibana_data:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - monitoring.kibana.collection.enabled=true + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + networks: + - libre-network + + redis: + container_name: redis + image: redis:7.0.11-alpine + restart: on-failure + volumes: + - redis:/data + ports: + - "6379:6379" + networks: + - libre-network + + redis-commander: + profiles: + - tools + image: rediscommander/redis-commander:latest + restart: unless-stopped + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8089:8081" + networks: + - libre-network + + rabbitmq: + container_name: rabbitmq + image: rabbitmq:3.11.15-management + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD} + - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST} + volumes: + - rabbitmq:/var/lib/rabbitmq + ports: + - "5672:5672" + - "15672:15672" + networks: + - libre-network + +volumes: + certs: + driver: local + es_data: + driver: local + kibana_data: + driver: local + rabbitmq: + driver: local + redis: + driver: local + +networks: + libre-network: + external: true diff --git a/run.sh b/run.sh index 6bd734a..f2699f9 100644 --- a/run.sh +++ b/run.sh @@ -46,7 +46,7 @@ stop_network() { genesis() { rm secrets/*.priv secrets/*.pub - docker run -d --name genesis --network libre-network -p 8888:8888 -p 9876:9876 -p 9879:9879 \ + docker run -d --name genesis --network libre-network -p 8888:8888 -p 8080:8080 -p 9876:9876 -p 9879:9879 \ -e BP_NAME=genesis \ -e INITIAL_CHAIN_ID=${INITIAL_CHAIN_ID} \ -e TESTNET_EOSIO_PRIVATE_KEY=${TESTNET_EOSIO_PRIVATE_KEY} \ @@ -86,6 +86,14 @@ start_single_node() { eoscostarica506/libre-node:latest bash -c "./start.sh" } +start_api_history() { + cd api-history/infra + docker-compose up -d + cd ../hyperion + docker-compose up -d + cd .. +} + create_network() { docker network create --driver bridge --subnet 192.168.0.0/24 libre-network }