From f063faae4ff8533486bab37508a45cbb5ea15d39 Mon Sep 17 00:00:00 2001 From: kiraum Date: Fri, 17 May 2024 11:31:04 +0200 Subject: [PATCH] feat: adding json file support and metaclass backoff for the main class / fix typo (#31) * fix: typo * feat: adding json file support and metaclass backoff for the main class --- .github/workflows/pylint.yml | 4 +- peering_gossip.py | 4 +- pgossip/pgossip.py | 107 ++++++++++++++++++++++++++++++----- pyproject.toml | 1 + requirements.txt | 11 ++-- 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c72b430..8415877 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -16,8 +16,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pylint requests pyaml + pip install --no-cache-dir -U pip uv + uv pip install --system --break-system-packages -r requirements.txt - name: Analysing the code with pylint run: | pylint $(git ls-files '*.py') diff --git a/peering_gossip.py b/peering_gossip.py index e4954a2..1c65723 100755 --- a/peering_gossip.py +++ b/peering_gossip.py @@ -34,12 +34,12 @@ def main(): pgossip = PGossip() if args.lg is not None: - pgossip.alice_hos(args.lg) + pgossip.alice_host(args.lg) if args.all is True: ixps = pgossip.load_yaml() for ixp in ixps["ixps"]: - pgossip.alice_hos(ixp) + pgossip.alice_host(ixp) if options is False: if len(sys.argv) == 1: diff --git a/pgossip/pgossip.py b/pgossip/pgossip.py index 4e457ca..3163558 100644 --- a/pgossip/pgossip.py +++ b/pgossip/pgossip.py @@ -5,31 +5,64 @@ import sys import time +import backoff import requests import yaml -class PGossip: +class RetryMeta(type): """ - Peering Buddy main functions for analyzing peering information. + A metaclass that applies retry logic to all callable methods of a class using backoff. - Provides methods for generating a hall of shame, fetching information about alive route servers - and their neighbors, retrieving ASN whois information, creating a report, and loading a YAML config file. + This metaclass enhances methods to retry up to 5 times upon `requests.exceptions.RequestException`, + using an exponential backoff strategy. It automatically decorates all methods that are not special + methods (not starting with '__'). Attributes: - None + name (str): The name of the class. + bases (tuple): The base classes of the class. + dct (dict): The dictionary containing the class's attributes. + + Returns: + type: The new class with modified methods. + """ + + def __new__(mcs, name, bases, dct): + for key, value in dct.items(): + if callable(value) and not key.startswith("__"): + dct[key] = backoff.on_exception( + backoff.expo, requests.exceptions.RequestException, max_tries=5 + )(value) + return type.__new__(mcs, name, bases, dct) + + +class PGossip(metaclass=RetryMeta): + """ + A class to handle peering and routing information gathering and reporting. + + This class provides methods to fetch and analyze routing data from specified URLs, + generate reports based on the gathered data, and manage interactions with external APIs + for data retrieval and report generation. Methods: - alice_hos: Generate hall of shame based on provided URL. - alice_rs: Get alive looking glass route servers. - alice_neighbours: Get alive looking glass neighbors for a specific route server. - bv_asn_whois: Return ASN whois information from BGPView API. - create_report: Create a pastebin-like report using glot.io API. - load_yaml: Load a YAML config file. + alice_host(url): Main method to process routing data for a given URL. + parse_text_to_json(data_text): Converts text data into JSON format. + write_report_to_file(fname, data, as_json): Writes data to a file in text or JSON format. + alice_rs(url): Fetches alive route servers from a given URL. + alice_neighbours(url, route_server): Fetches routing neighbors for a specific route server. + bv_asn_whois(asn): Retrieves ASN WHOIS information from the BGPView API. + create_report(data): Creates a pastebin-like report. + load_yaml(): Loads a YAML configuration file. + + Attributes: + None explicitly defined; configuration and state are managed internally within methods. """ + def __init__(self): + pass + # pylint: disable=too-many-locals - def alice_hos(self, url): + def alice_host(self, url): """ Generate hall of shame based on provided URL. @@ -87,10 +120,56 @@ def alice_hos(self, url): report_link = self.create_report("\n".join(map(str, text))) print("=" * 80) print(f"We created a sharable report link, enjoy => {report_link}") - fwrite = f"reports/{fname}.txt" + self.write_report_to_file(fname, "\n".join(map(str, text)), as_json=False) + self.write_report_to_file(fname, "\n".join(map(str, text)), as_json=True) + + def parse_text_to_json(self, data_text): + """ + Convert a list of delimited text data into a list of dictionaries. + + Args: + data_text (str): A string containing multiple lines of data, each line is a delimited record. + + Returns: + list: A list of dictionaries with parsed data. + """ + lines = data_text.strip().split("\n") + headers = [header.strip() for header in lines[0].split("|")] + json_data = [] + + for line in lines[1:]: + values = [value.strip() for value in line.split("|")] + entry = dict(zip(headers, values)) + json_data.append(entry) + return json_data + + def write_report_to_file(self, fname: str, data: list, as_json: bool = False): + """ + Write data to a file, creating the necessary directories if they do not exist. + The data can be written as plain text or as JSON. + + Args: + fname (str): The filename (without extension) where the data will be saved. + data (list): A list of data entries, each entry can be a string or a dictionary. + as_json (bool): If True, writes the data in JSON format. Otherwise, writes as plain text. + + Example: + write_report_to_file("2023-01-01_report", data, as_json=True) + """ + # Construct the full file path with directory and filename + extension = "json" if as_json else "txt" + fwrite = f"reports/{fname}.{extension}" + + # Ensure the directory exists; if not, create it os.makedirs(os.path.dirname(fwrite), exist_ok=True) + + # Open the file and write the data to it with open(fwrite, "w", encoding="utf8") as tfile: - tfile.write("\n".join(map(str, text))) + if as_json: + data = self.parse_text_to_json(data) + json.dump(data, tfile, indent=4) + else: + tfile.write(data) def alice_rs(self, url): """ diff --git a/pyproject.toml b/pyproject.toml index 4145696..813598e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ requires-python = ">=3.12" keywords = ["peering", "tool", "shame", "hall"] dependencies = [ "requests", + "backoff", "black", "isort", "pylint", diff --git a/requirements.txt b/requirements.txt index d699073..ae26f12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml -astroid==3.1.0 +astroid==3.2.1 # via pylint +backoff==2.2.1 black==24.4.2 certifi==2024.2.2 # via requests @@ -29,21 +30,21 @@ pathspec==0.12.1 # via # black # yamllint -platformdirs==4.2.1 +platformdirs==4.2.2 # via # black # pylint pluggy==1.5.0 # via pytest -pylint==3.1.0 +pylint==3.2.0 pytest==8.2.0 pyyaml==6.0.1 # via yamllint requests==2.31.0 -ruff==0.4.3 +ruff==0.4.4 tomlkit==0.12.5 # via pylint urllib3==2.2.1 # via requests -uv==0.1.41 +uv==0.1.44 yamllint==1.35.1