From 16ab9e6e9f41941c85d07ee8211d50f21043a568 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Sat, 27 Apr 2024 14:01:41 -0400 Subject: [PATCH 1/4] Add gap finding python tool for block height gaps --- tools/find-block-height-gaps/.gitignore | 3 + tools/find-block-height-gaps/main.py | 71 +++++++++++++++++++ tools/find-block-height-gaps/requirements.txt | 1 + 3 files changed, 75 insertions(+) create mode 100644 tools/find-block-height-gaps/.gitignore create mode 100644 tools/find-block-height-gaps/main.py create mode 100644 tools/find-block-height-gaps/requirements.txt diff --git a/tools/find-block-height-gaps/.gitignore b/tools/find-block-height-gaps/.gitignore new file mode 100644 index 0000000..3f9a892 --- /dev/null +++ b/tools/find-block-height-gaps/.gitignore @@ -0,0 +1,3 @@ +.env +venv +output/* diff --git a/tools/find-block-height-gaps/main.py b/tools/find-block-height-gaps/main.py new file mode 100644 index 0000000..ed8be45 --- /dev/null +++ b/tools/find-block-height-gaps/main.py @@ -0,0 +1,71 @@ +import psycopg2 +import os +import json +import traceback +import argparse + +def get_env(): + ret = { + "host": os.environ.get("DB_HOST", ""), + "password": os.environ.get("DB_PASSWORD", ""), + "user": os.environ.get("DB_USER", ""), + "port": os.environ.get("DB_PORT", ""), + "db_name": os.environ.get("DB_NAME", "") + } + + if any([ret[x] == "" for x in ret]): + raise Exception("Must provide env vars") + + return ret + +def get_chain_id_arg(): + parser = argparse.ArgumentParser(description="Dump failed block heights") + parser.add_argument("--chain-id", type=str, default="osmosis-1", help="Chain ID to dump failed block heights from") + args = parser.parse_args() + return args.chain_id + +SELECT_CHAINS_QUERY = "SELECT id FROM chains WHERE chain_id=%s;" +GAPS_QUERY = """ +SELECT height + 1 AS gap_start, + next_height - 1 AS gap_end +FROM ( + SELECT height, + LEAD(height) OVER (ORDER BY height) AS next_height + FROM blocks WHERE blockchain_id = %s +) nr +WHERE height + 1 <> next_height; +""" + +if __name__ == "__main__": + env = get_env() + chain_id = get_chain_id_arg() + os.makedirs("./output", exist_ok=True) + + conn = psycopg2.connect(f"dbname={env['db_name']} user={env['user']} host={env['host']} password={env['password']} port={env['port']}") + + try: + + rec = None + with conn.cursor() as cur: + cur.execute(SELECT_CHAINS_QUERY, (chain_id,)) + rec = cur.fetchone() + if rec is None: + raise Exception(f"Chain ID {chain_id} not found") + + with conn.cursor() as cur: + cur.execute(GAPS_QUERY, (rec[0],)) + gaps = cur.fetchall() + + # flatten and fill the groups to get a list of missing heights + flat = [] + for gap in gaps: + flattened_and_filled = [x for x in range(gap[0], gap[1] + 1)] + flat.extend(flattened_and_filled) + + json.dump(gaps, open("output/gaps.json", 'w'), indent=4) + json.dump(flat, open("output/missing_heights.json", 'w'), indent=4) + except Exception as err: + print(err) + traceback.print_exc() + finally: + conn.close() \ No newline at end of file diff --git a/tools/find-block-height-gaps/requirements.txt b/tools/find-block-height-gaps/requirements.txt new file mode 100644 index 0000000..9edde47 --- /dev/null +++ b/tools/find-block-height-gaps/requirements.txt @@ -0,0 +1 @@ +psycopg2==2.9.7 From c56c4d1036d096001459ea612acbfd83b6d81d10 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Sat, 27 Apr 2024 14:33:10 -0400 Subject: [PATCH 2/4] Add -ff flag to enable flatten and fill since this is the bottleneck --- tools/find-block-height-gaps/main.py | 33 +++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/find-block-height-gaps/main.py b/tools/find-block-height-gaps/main.py index ed8be45..47a2063 100644 --- a/tools/find-block-height-gaps/main.py +++ b/tools/find-block-height-gaps/main.py @@ -18,11 +18,12 @@ def get_env(): return ret -def get_chain_id_arg(): +def get_args(): parser = argparse.ArgumentParser(description="Dump failed block heights") parser.add_argument("--chain-id", type=str, default="osmosis-1", help="Chain ID to dump failed block heights from") + parser.add_argument("--flatten-and-fill", "-ff", action="store_true", help="Flatten and fill the gaps and output to a file") args = parser.parse_args() - return args.chain_id + return args SELECT_CHAINS_QUERY = "SELECT id FROM chains WHERE chain_id=%s;" GAPS_QUERY = """ @@ -36,9 +37,18 @@ def get_chain_id_arg(): WHERE height + 1 <> next_height; """ +def flatten_and_fill(gaps): + flat = [] + for gap in gaps: + flattened_and_filled = [x for x in range(gap[0], gap[1] + 1)] + flat.extend(flattened_and_filled) + return flat + if __name__ == "__main__": env = get_env() - chain_id = get_chain_id_arg() + args = get_args() + chain_id = args.chain_id + ff = args.flatten_and_fill os.makedirs("./output", exist_ok=True) conn = psycopg2.connect(f"dbname={env['db_name']} user={env['user']} host={env['host']} password={env['password']} port={env['port']}") @@ -46,24 +56,27 @@ def get_chain_id_arg(): try: rec = None + print(f"Finding chain ID {chain_id}...") with conn.cursor() as cur: cur.execute(SELECT_CHAINS_QUERY, (chain_id,)) rec = cur.fetchone() if rec is None: raise Exception(f"Chain ID {chain_id} not found") - + + print("Executing gap finder...") with conn.cursor() as cur: cur.execute(GAPS_QUERY, (rec[0],)) gaps = cur.fetchall() - # flatten and fill the groups to get a list of missing heights - flat = [] - for gap in gaps: - flattened_and_filled = [x for x in range(gap[0], gap[1] + 1)] - flat.extend(flattened_and_filled) json.dump(gaps, open("output/gaps.json", 'w'), indent=4) - json.dump(flat, open("output/missing_heights.json", 'w'), indent=4) + + if ff and len(gaps) > 0: + print("Found gaps, flattening and filling...") + flat = flatten_and_fill(gaps) + json.dump(flat, open("output/missing_heights.json", 'w'), indent=4) + + print("Done") except Exception as err: print(err) traceback.print_exc() From e1c4a7cb0936346a25acae7a627c6c315d2e49de Mon Sep 17 00:00:00 2001 From: pharr117 Date: Sat, 27 Apr 2024 14:34:35 -0400 Subject: [PATCH 3/4] Update text --- tools/find-block-height-gaps/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/find-block-height-gaps/main.py b/tools/find-block-height-gaps/main.py index 47a2063..a932afc 100644 --- a/tools/find-block-height-gaps/main.py +++ b/tools/find-block-height-gaps/main.py @@ -19,9 +19,9 @@ def get_env(): return ret def get_args(): - parser = argparse.ArgumentParser(description="Dump failed block heights") + parser = argparse.ArgumentParser(description="Finds and outputs block height gaps in a chain") parser.add_argument("--chain-id", type=str, default="osmosis-1", help="Chain ID to dump failed block heights from") - parser.add_argument("--flatten-and-fill", "-ff", action="store_true", help="Flatten and fill the gaps and output to a file") + parser.add_argument("--flatten-and-fill", "-ff", action="store_true", help="Flatten and fill the gaps and output to a file next to the gaps file") args = parser.parse_args() return args From c68d9e92c12c0af946267b085cd898da98499582 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Sat, 27 Apr 2024 14:47:11 -0400 Subject: [PATCH 4/4] Precommit --- tools/find-block-height-gaps/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/find-block-height-gaps/main.py b/tools/find-block-height-gaps/main.py index a932afc..2b4c41a 100644 --- a/tools/find-block-height-gaps/main.py +++ b/tools/find-block-height-gaps/main.py @@ -27,10 +27,10 @@ def get_args(): SELECT_CHAINS_QUERY = "SELECT id FROM chains WHERE chain_id=%s;" GAPS_QUERY = """ -SELECT height + 1 AS gap_start, +SELECT height + 1 AS gap_start, next_height - 1 AS gap_end FROM ( - SELECT height, + SELECT height, LEAD(height) OVER (ORDER BY height) AS next_height FROM blocks WHERE blockchain_id = %s ) nr @@ -62,7 +62,7 @@ def flatten_and_fill(gaps): rec = cur.fetchone() if rec is None: raise Exception(f"Chain ID {chain_id} not found") - + print("Executing gap finder...") with conn.cursor() as cur: cur.execute(GAPS_QUERY, (rec[0],)) @@ -81,4 +81,4 @@ def flatten_and_fill(gaps): print(err) traceback.print_exc() finally: - conn.close() \ No newline at end of file + conn.close()