From af2ccc0e0b1047c24cee45425761eb5d1727b042 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 19:24:16 +0200 Subject: [PATCH 01/10] estimated_upgrade_time, upgrade_name --- app.py | 120 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 58 insertions(+), 62 deletions(-) diff --git a/app.py b/app.py index 3e5abe5..13d83e5 100644 --- a/app.py +++ b/app.py @@ -1,10 +1,11 @@ import requests import re from datetime import datetime +from datetime import timedelta from random import shuffle # import logging import threading -from flask import Flask, jsonify, request +from flask import Flask, jsonify, request, Response from flask_caching import Cache from concurrent.futures import ThreadPoolExecutor from time import sleep @@ -30,7 +31,7 @@ repo_retain_hours = int(os.environ.get('REPO_RETAIN_HOURS', 3)) # Initialize number of workers -num_workers = int(os.environ.get('NUM_WORKERS', 10)) +num_workers = int(os.environ.get('NUM_WORKERS', 24)) GITHUB_API_BASE_URL = "https://api.github.com/repos/cosmos/chain-registry/contents" @@ -87,46 +88,6 @@ def download_and_extract_repo(): return repo_path -def fetch_directory_names(path): - headers = {'Accept': 'application/vnd.github.v3+json'} - url = f"{GITHUB_API_BASE_URL}/{path}" - response = requests.get(url, headers=headers) - response_data = response.json() - - if isinstance(response_data, list): - dir_names = [item['name'] for item in response_data if item['type'] == 'dir' and not item['name'].startswith(('.', '_'))] - return dir_names - else: - return [] - -def read_chain_json_from_local(network, base_path): - """Read the chain.json file for a given network from the local repository.""" - chain_json_path = os.path.join(base_path, network, 'chain.json') - with open(chain_json_path, 'r') as f: - return json.load(f) - -def process_data_for_network(network, full_data, network_type): - """Process the data for a specific network.""" - network_data = full_data.get(network, []) - file_names = [item['name'] for item in network_data if item['type'] == 'file'] - - return { - "type": network_type, - "network": network, - "files": file_names - } - -def process_data_for_local_network(network, base_path, network_type): - """Process the data for a specific network from local files.""" - network_path = os.path.join(base_path, network) - file_names = [f for f in os.listdir(network_path) if os.path.isfile(os.path.join(network_path, f))] - - return { - "type": network_type, - "network": network, - "files": file_names - } - def get_healthy_rpc_endpoints(rpc_endpoints): with ThreadPoolExecutor(max_workers=num_workers) as executor: healthy_rpc_endpoints = [rpc for rpc, is_healthy in executor.map(lambda rpc: (rpc, is_endpoint_healthy(rpc['address'])), rpc_endpoints) if is_healthy] @@ -162,7 +123,7 @@ def check_rest_endpoint(rest_url): """Check the REST endpoint and return the application version and response time.""" start_time = datetime.now() try: - response = requests.get(f"{rest_url}/node_info", timeout=3, verify=False) + response = requests.get(f"{rest_url}/node_info", timeout=1, verify=False) response.raise_for_status() elapsed_time = (datetime.now() - start_time).total_seconds() @@ -175,12 +136,28 @@ def check_rest_endpoint(rest_url): def get_latest_block_height_rpc(rpc_url): """Fetch the latest block height from the RPC endpoint.""" try: - response = requests.get(f"{rpc_url}/status", timeout=3) + response = requests.get(f"{rpc_url}/status", timeout=1) response.raise_for_status() data = response.json() return int(data.get('result', {}).get('sync_info', {}).get('latest_block_height', 0)) except requests.RequestException as e: return -1 # Return -1 to indicate an error + +def get_block_time_rpc(rpc_url, height): + """Fetch the block header time for a given block height from the RPC endpoint.""" + try: + response = requests.get(f"{rpc_url}/block?height={height}", timeout=1) + response.raise_for_status() + data = response.json() + return data.get('result', {}).get('block', {}).get('header', {}).get('time', "") + except requests.RequestException as e: + return None + +def parse_isoformat_string(date_string): + date_string = re.sub(r"(\.\d{6})\d+Z", r"\1Z", date_string) + date_string = date_string.replace("Z", "+00:00") + return datetime.fromisoformat(date_string) + def fetch_all_endpoints(network_type, base_url, request_data): """Fetch all the REST and RPC endpoints for all networks and store in a map.""" @@ -246,8 +223,8 @@ def fetch_active_upgrade_proposals(rest_url): height = 0 if version: - return version, height - return None, None + return plan_name, version, height + return None, None, None except requests.RequestException as e: print(f"Error received from server {rest_url}: {e}") raise e @@ -263,15 +240,16 @@ def fetch_current_upgrade_plan(rest_url): plan = data.get("plan", {}) if plan: - version_match = SEMANTIC_VERSION_PATTERN.search(plan.get("name", "")) + plan_name = plan.get("name", "") + version_match = SEMANTIC_VERSION_PATTERN.search(plan_name) if version_match: version = version_match.group(1) try: height = int(plan.get("height", 0)) except ValueError: height = 0 - return version, height - return None, None + return plan_name, version, height + return None, None, None except requests.RequestException as e: print(f"Error received from server {rest_url}: {e}") raise e @@ -305,7 +283,6 @@ def fetch_data_for_network(network, network_type): print(f"Found {len(rest_endpoints)} rest endpoints and {len(rpc_endpoints)} rpc endpoints for {network}") # Prioritize RPC endpoints for fetching the latest block height - # Shuffle RPC endpoints to avoid calling the same one over and over latest_block_height = -1 healthy_rpc_endpoints = get_healthy_rpc_endpoints(rpc_endpoints) healthy_rest_endpoints = get_healthy_rest_endpoints(rest_endpoints) @@ -322,9 +299,8 @@ def fetch_data_for_network(network, network_type): break # Check for active upgrade proposals - # Shuffle RPC endpoints to avoid calling the same one over and over - shuffle(healthy_rest_endpoints) upgrade_block_height = None + upgrade_name = "" upgrade_version = "" source = "" @@ -334,8 +310,8 @@ def fetch_data_for_network(network, network_type): if current_endpoint in SERVER_BLACKLIST: continue try: - active_upgrade_version, active_upgrade_height = fetch_active_upgrade_proposals(current_endpoint) - current_upgrade_version, current_upgrade_height = fetch_current_upgrade_plan(current_endpoint) + active_upgrade_name, active_upgrade_version, active_upgrade_height = fetch_active_upgrade_proposals(current_endpoint) + current_upgrade_name, current_upgrade_version, current_upgrade_height = fetch_current_upgrade_plan(current_endpoint) except: if index + 1 < len(healthy_rest_endpoints): print(f"Failed to query rest endpoints {current_endpoint}, trying next rest endpoint") @@ -347,24 +323,45 @@ def fetch_data_for_network(network, network_type): if active_upgrade_version and (active_upgrade_height is not None) and active_upgrade_height > latest_block_height: upgrade_block_height = active_upgrade_height upgrade_version = active_upgrade_version + upgrade_name = active_upgrade_name source = "active_upgrade_proposals" break if current_upgrade_version and (current_upgrade_height is not None) and current_upgrade_height > latest_block_height: upgrade_block_height = current_upgrade_height upgrade_version = current_upgrade_version + upgrade_name = current_upgrade_name source = "current_upgrade_plan" break + # Calculate average block time + current_block_time = get_block_time_rpc(rpc_server_used, latest_block_height) + past_block_time = get_block_time_rpc(rpc_server_used, latest_block_height - 10000) + avg_block_time_seconds = None + + if current_block_time and past_block_time: + current_block_datetime = parse_isoformat_string(current_block_time) + past_block_datetime = parse_isoformat_string(past_block_time) + avg_block_time_seconds = (current_block_datetime - past_block_datetime).total_seconds() / 10000 + + # Estimate the upgrade time + estimated_upgrade_time = None + if upgrade_block_height and avg_block_time_seconds: + estimated_seconds_until_upgrade = avg_block_time_seconds * (upgrade_block_height - latest_block_height) + estimated_upgrade_datetime = datetime.utcnow() + timedelta(seconds=estimated_seconds_until_upgrade) + estimated_upgrade_time = estimated_upgrade_datetime.isoformat().replace('+00:00', 'Z') + output_data = { - "type": network_type, "network": network, - "upgrade_found": upgrade_version != "", + "type": network_type, + "rpc_server": rpc_server_used, "latest_block_height": latest_block_height, + "upgrade_found": upgrade_version != "", + "upgrade_name": upgrade_name, + "source": source, "upgrade_block_height": upgrade_block_height, - "version": upgrade_version, - "rpc_server": rpc_server_used, - "source": source + "estimated_upgrade_time": estimated_upgrade_time, + "version": upgrade_version } print(f"Completed fetch data for network {network}") return output_data @@ -418,11 +415,9 @@ def update_data(): def start_update_data_thread(): - print("Starting the update_data thread...") update_thread = threading.Thread(target=update_data) update_thread.daemon = True update_thread.start() - print("update_data thread started.") @app.route('/healthz') def health_check(): @@ -458,7 +453,8 @@ def fetch_network_data(): # Sort the results by 'upgrade_found' in descending order (chain upgrades first) sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) - return jsonify(sorted_results) + response = Response(json.dumps(sorted_results, indent=2), content_type="application/json") + return response except Exception as e: return jsonify({"error": str(e)}), 500 From 5ef03548f5a21e9276913548b540384ebb3e942c Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 19:24:37 +0200 Subject: [PATCH 02/10] set default workers to 10 --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 13d83e5..d1c5594 100644 --- a/app.py +++ b/app.py @@ -31,7 +31,7 @@ repo_retain_hours = int(os.environ.get('REPO_RETAIN_HOURS', 3)) # Initialize number of workers -num_workers = int(os.environ.get('NUM_WORKERS', 24)) +num_workers = int(os.environ.get('NUM_WORKERS', 10)) GITHUB_API_BASE_URL = "https://api.github.com/repos/cosmos/chain-registry/contents" From 3bd2cefc471e88a9824eb07b0bf848f491242cab Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 19:36:24 +0200 Subject: [PATCH 03/10] consistent output ordering --- app.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index d1c5594..3528925 100644 --- a/app.py +++ b/app.py @@ -158,6 +158,24 @@ def parse_isoformat_string(date_string): date_string = date_string.replace("Z", "+00:00") return datetime.fromisoformat(date_string) +def reorder_data(data): + """ + Reorder the data according to the specified structure. + + :param data: Original data dictionary. + :return: Reordered data dictionary. + """ + return { + "type": data.get("type"), + "network": data.get("network"), + "rpc_server": data.get("rpc_server"), + "latest_block_height": data.get("latest_block_height"), + "upgrade_found": data.get("upgrade_found"), + "source": data.get("source"), + "upgrade_block_height": data.get("upgrade_block_height"), + "estimated_upgrade_time": data.get("estimated_upgrade_time"), + "version": data.get("version") # Assuming "upgrade_name" is the version + } def fetch_all_endpoints(network_type, base_url, request_data): """Fetch all the REST and RPC endpoints for all networks and store in a map.""" @@ -450,11 +468,9 @@ def fetch_network_data(): filtered_testnet_data = [data for data in testnet_data if data['network'] in request_data.get("TESTNETS", [])] results = filtered_mainnet_data + filtered_testnet_data - # Sort the results by 'upgrade_found' in descending order (chain upgrades first) - sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) - - response = Response(json.dumps(sorted_results, indent=2), content_type="application/json") - return response + reordered_results = [reorder_data(result) for result in results] + sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) + return sorted_results except Exception as e: return jsonify({"error": str(e)}), 500 @@ -466,10 +482,9 @@ def get_mainnet_data(): if results is None: return jsonify({"error": "Data not available"}), 500 - # Filter out None values from results results = [r for r in results if r is not None] - - sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) + reordered_results = [reorder_data(result) for result in results] + sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) return jsonify(sorted_results) @app.route('/testnets') @@ -479,10 +494,9 @@ def get_testnet_data(): if results is None: return jsonify({"error": "Data not available"}), 500 - # Filter out None values from results results = [r for r in results if r is not None] - - sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) + reordered_results = [reorder_data(result) for result in results] + sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) return jsonify(sorted_results) if __name__ == '__main__': From 6ac3057ba1326d9307f0be7fb0d122c917cb1271 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 19:46:32 +0200 Subject: [PATCH 04/10] disable flask cache --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 3528925..4d2bc15 100644 --- a/app.py +++ b/app.py @@ -476,7 +476,7 @@ def fetch_network_data(): return jsonify({"error": str(e)}), 500 @app.route('/mainnets') -@cache.cached(timeout=600) # Cache the result for 10 minutes +# @cache.cached(timeout=600) # Cache the result for 10 minutes def get_mainnet_data(): results = cache.get('MAINNET_DATA') if results is None: @@ -488,7 +488,7 @@ def get_mainnet_data(): return jsonify(sorted_results) @app.route('/testnets') -@cache.cached(timeout=600) # Cache the result for 10 minutes +# @cache.cached(timeout=600) # Cache the result for 10 minutes def get_testnet_data(): results = cache.get('TESTNET_DATA') if results is None: From a9fdebe13db72c985a0c228611ba87e45c886e9b Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 19:51:36 +0200 Subject: [PATCH 05/10] jsonify fetch response --- app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 4d2bc15..9456cf5 100644 --- a/app.py +++ b/app.py @@ -470,7 +470,10 @@ def fetch_network_data(): reordered_results = [reorder_data(result) for result in results] sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) - return sorted_results + return jsonify(sorted_results) + + except Exception as e: + return jsonify({"error": str(e)}), 500 except Exception as e: return jsonify({"error": str(e)}), 500 From 2053a4525ad7fc5df40f47768994ebbf4497439b Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 20:00:05 +0200 Subject: [PATCH 06/10] order params after sort --- app.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index 9456cf5..56ac6a1 100644 --- a/app.py +++ b/app.py @@ -174,7 +174,7 @@ def reorder_data(data): "source": data.get("source"), "upgrade_block_height": data.get("upgrade_block_height"), "estimated_upgrade_time": data.get("estimated_upgrade_time"), - "version": data.get("version") # Assuming "upgrade_name" is the version + "version": data.get("version") } def fetch_all_endpoints(network_type, base_url, request_data): @@ -468,9 +468,9 @@ def fetch_network_data(): filtered_testnet_data = [data for data in testnet_data if data['network'] in request_data.get("TESTNETS", [])] results = filtered_mainnet_data + filtered_testnet_data - reordered_results = [reorder_data(result) for result in results] - sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) - return jsonify(sorted_results) + sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) + reordered_results = [reorder_data(result) for result in sorted_results] + return jsonify(reordered_results) except Exception as e: return jsonify({"error": str(e)}), 500 @@ -486,9 +486,9 @@ def get_mainnet_data(): return jsonify({"error": "Data not available"}), 500 results = [r for r in results if r is not None] - reordered_results = [reorder_data(result) for result in results] - sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) - return jsonify(sorted_results) + sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) + reordered_results = [reorder_data(result) for result in sorted_results] + return jsonify(reordered_results) @app.route('/testnets') # @cache.cached(timeout=600) # Cache the result for 10 minutes @@ -498,9 +498,9 @@ def get_testnet_data(): return jsonify({"error": "Data not available"}), 500 results = [r for r in results if r is not None] - reordered_results = [reorder_data(result) for result in results] - sorted_results = sorted(reordered_results, key=lambda x: x['upgrade_found'], reverse=True) - return jsonify(sorted_results) + sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) + reordered_results = [reorder_data(result) for result in sorted_results] + return jsonify(reordered_results) if __name__ == '__main__': app.debug = True From 9ff224d872dcdd7f219c8caefe9264e830aadba5 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 20:08:57 +0200 Subject: [PATCH 07/10] add upgrade_name to response --- app.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index 56ac6a1..218bf7e 100644 --- a/app.py +++ b/app.py @@ -159,18 +159,13 @@ def parse_isoformat_string(date_string): return datetime.fromisoformat(date_string) def reorder_data(data): - """ - Reorder the data according to the specified structure. - - :param data: Original data dictionary. - :return: Reordered data dictionary. - """ return { - "type": data.get("type"), "network": data.get("network"), + "type": data.get("type"), "rpc_server": data.get("rpc_server"), "latest_block_height": data.get("latest_block_height"), "upgrade_found": data.get("upgrade_found"), + "upgrade_name": data.get("upgrade_name"), "source": data.get("source"), "upgrade_block_height": data.get("upgrade_block_height"), "estimated_upgrade_time": data.get("estimated_upgrade_time"), From d1569c20474b56b8eb5118008e6b6b3bdeb91dd4 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 20:14:57 +0200 Subject: [PATCH 08/10] order correctly --- app.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 218bf7e..6eac000 100644 --- a/app.py +++ b/app.py @@ -9,6 +9,7 @@ from flask_caching import Cache from concurrent.futures import ThreadPoolExecutor from time import sleep +from collections import OrderedDict import os import zipfile import json @@ -159,18 +160,19 @@ def parse_isoformat_string(date_string): return datetime.fromisoformat(date_string) def reorder_data(data): - return { - "network": data.get("network"), - "type": data.get("type"), - "rpc_server": data.get("rpc_server"), - "latest_block_height": data.get("latest_block_height"), - "upgrade_found": data.get("upgrade_found"), - "upgrade_name": data.get("upgrade_name"), - "source": data.get("source"), - "upgrade_block_height": data.get("upgrade_block_height"), - "estimated_upgrade_time": data.get("estimated_upgrade_time"), - "version": data.get("version") - } + ordered_data = OrderedDict([ + ("type", data.get("type")), + ("network", data.get("network")), + ("rpc_server", data.get("rpc_server")), + ("latest_block_height", data.get("latest_block_height")), + ("upgrade_found", data.get("upgrade_found")), + ("upgrade_name", data.get("upgrade_name")), + ("source", data.get("source")), + ("upgrade_block_height", data.get("upgrade_block_height")), + ("estimated_upgrade_time", data.get("estimated_upgrade_time")), + ("version", data.get("version")) + ]) + return ordered_data def fetch_all_endpoints(network_type, base_url, request_data): """Fetch all the REST and RPC endpoints for all networks and store in a map.""" From 4123cd475bba6dcddaf69790dfe277c805e9f33b Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 20:22:22 +0200 Subject: [PATCH 09/10] order params correctly --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 6eac000..ffccef0 100644 --- a/app.py +++ b/app.py @@ -467,7 +467,7 @@ def fetch_network_data(): sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return jsonify(reordered_results) + return Response(json.dumps(reordered_results, indent=2), content_type="application/json") except Exception as e: return jsonify({"error": str(e)}), 500 @@ -485,7 +485,7 @@ def get_mainnet_data(): results = [r for r in results if r is not None] sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return jsonify(reordered_results) + return Response(json.dumps(reordered_results), content_type="application/json") @app.route('/testnets') # @cache.cached(timeout=600) # Cache the result for 10 minutes @@ -497,7 +497,7 @@ def get_testnet_data(): results = [r for r in results if r is not None] sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return jsonify(reordered_results) + return Response(json.dumps(reordered_results), content_type="application/json") if __name__ == '__main__': app.debug = True From c2f861a1ddb936c5765e543150d45921506b7377 Mon Sep 17 00:00:00 2001 From: clemensgg Date: Sat, 2 Sep 2023 20:31:26 +0200 Subject: [PATCH 10/10] add newline after response --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index ffccef0..b6c1131 100644 --- a/app.py +++ b/app.py @@ -467,7 +467,7 @@ def fetch_network_data(): sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return Response(json.dumps(reordered_results, indent=2), content_type="application/json") + return Response(json.dumps(reordered_results, indent=2) + '\n', content_type="application/json") except Exception as e: return jsonify({"error": str(e)}), 500 @@ -485,7 +485,7 @@ def get_mainnet_data(): results = [r for r in results if r is not None] sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return Response(json.dumps(reordered_results), content_type="application/json") + return Response(json.dumps(reordered_results) + '\n', content_type="application/json") @app.route('/testnets') # @cache.cached(timeout=600) # Cache the result for 10 minutes @@ -497,7 +497,7 @@ def get_testnet_data(): results = [r for r in results if r is not None] sorted_results = sorted(results, key=lambda x: x['upgrade_found'], reverse=True) reordered_results = [reorder_data(result) for result in sorted_results] - return Response(json.dumps(reordered_results), content_type="application/json") + return Response(json.dumps(reordered_results) + '\n', content_type="application/json") if __name__ == '__main__': app.debug = True