From 781c8c09f43bcd5f1a1599da60923009baba3ee4 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 01:47:58 +0100 Subject: [PATCH 1/9] Refactoring, test coverage increased to 60% --- .github/workflows/python-package.yml | 15 +++- maigret/checking.py | 111 +++++++++++++++------------ maigret/errors.py | 48 +++++++++++- maigret/maigret.py | 32 +------- maigret/notify.py | 10 +-- maigret/report.py | 12 +-- maigret/resources/data.json | 2 +- maigret/result.py | 17 ++-- maigret/submit.py | 10 ++- tests/conftest.py | 20 +++++ tests/test_maigret.py | 32 ++------ tests/test_notify.py | 18 ++--- tests/test_report.py | 10 +-- utils/import_sites.py | 10 +-- 14 files changed, 192 insertions(+), 155 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 78876c2b..e83b38ac 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,8 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -26,6 +27,14 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry python -m poetry install --with dev - - name: Test with pytest + - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - poetry run pytest --reruns 3 --reruns-delay 5 + coverage run -m pytest --reruns 3 --reruns-delay 5 + # Fail if coverage is less than 60% + coverage report --fail-under=60 + coverage html + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: htmlcov + path: htmlcov \ No newline at end of file diff --git a/maigret/checking.py b/maigret/checking.py index 738d3aa1..f6224830 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -31,7 +31,7 @@ AsyncioSimpleExecutor, AsyncioProgressbarQueueExecutor, ) -from .result import QueryResult, QueryStatus +from .result import MaigretCheckResult, MaigretCheckStatus from .sites import MaigretDatabase, MaigretSite from .types import QueryOptions, QueryResultWrapper from .utils import ascii_data_display, get_random_user_agent @@ -322,7 +322,7 @@ def process_site_result( break def build_result(status, **kwargs): - return QueryResult( + return MaigretCheckResult( username, site_name, url, @@ -334,11 +334,11 @@ def build_result(status, **kwargs): if check_error: logger.warning(check_error) - result = QueryResult( + result = MaigretCheckResult( username, site_name, url, - QueryStatus.UNKNOWN, + MaigretCheckStatus.UNKNOWN, query_time=response_time, error=check_error, context=str(CheckError), @@ -350,15 +350,15 @@ def build_result(status, **kwargs): [(absence_flag in html_text) for absence_flag in site.absence_strs] ) if not is_absence_detected and is_presense_detected: - result = build_result(QueryStatus.CLAIMED) + result = build_result(MaigretCheckStatus.CLAIMED) else: - result = build_result(QueryStatus.AVAILABLE) + result = build_result(MaigretCheckStatus.AVAILABLE) elif check_type in "status_code": # Checks if the status code of the response is 2XX if 200 <= status_code < 300: - result = build_result(QueryStatus.CLAIMED) + result = build_result(MaigretCheckStatus.CLAIMED) else: - result = build_result(QueryStatus.AVAILABLE) + result = build_result(MaigretCheckStatus.AVAILABLE) elif check_type == "response_url": # For this detection method, we have turned off the redirect. # So, there is no need to check the response URL: it will always @@ -366,9 +366,9 @@ def build_result(status, **kwargs): # code indicates that the request was successful (i.e. no 404, or # forward to some odd redirect). if 200 <= status_code < 300 and is_presense_detected: - result = build_result(QueryStatus.CLAIMED) + result = build_result(MaigretCheckStatus.CLAIMED) else: - result = build_result(QueryStatus.AVAILABLE) + result = build_result(MaigretCheckStatus.AVAILABLE) else: # It should be impossible to ever get here... raise ValueError( @@ -377,33 +377,11 @@ def build_result(status, **kwargs): extracted_ids_data = {} - if is_parsing_enabled and result.status == QueryStatus.CLAIMED: - try: - extracted_ids_data = extract(html_text) - except Exception as e: - logger.warning(f"Error while parsing {site.name}: {e}", exc_info=True) - + if is_parsing_enabled and result.status == MaigretCheckStatus.CLAIMED: + extracted_ids_data = extract_ids_data(html_text, logger, site) if extracted_ids_data: - new_usernames = {} - for k, v in extracted_ids_data.items(): - if "username" in k and not "usernames" in k: - new_usernames[v] = "username" - elif "usernames" in k: - try: - tree = ast.literal_eval(v) - if type(tree) == list: - for n in tree: - new_usernames[n] = "username" - except Exception as e: - logger.warning(e) - if k in SUPPORTED_IDS: - new_usernames[v] = k - - results_info["ids_usernames"] = new_usernames - links = ascii_data_display(extracted_ids_data.get("links", "[]")) - if "website" in extracted_ids_data: - links.append(extracted_ids_data["website"]) - results_info["ids_links"] = links + new_usernames = parse_usernames(extracted_ids_data, logger) + results_info = update_results_info(results_info, extracted_ids_data, new_usernames) result.ids_data = extracted_ids_data # Save status of request @@ -462,29 +440,29 @@ def make_site_result( # site check is disabled if site.disabled and not options['forced']: logger.debug(f"Site {site.name} is disabled, skipping...") - results_site["status"] = QueryResult( + results_site["status"] = MaigretCheckResult( username, site.name, url, - QueryStatus.ILLEGAL, + MaigretCheckStatus.ILLEGAL, error=CheckError("Check is disabled"), ) # current username type could not be applied elif site.type != options["id_type"]: - results_site["status"] = QueryResult( + results_site["status"] = MaigretCheckResult( username, site.name, url, - QueryStatus.ILLEGAL, + MaigretCheckStatus.ILLEGAL, error=CheckError('Unsupported identifier type', f'Want "{site.type}"'), ) # username is not allowed. elif site.regex_check and re.search(site.regex_check, username) is None: - results_site["status"] = QueryResult( + results_site["status"] = MaigretCheckResult( username, site.name, url, - QueryStatus.ILLEGAL, + MaigretCheckStatus.ILLEGAL, error=CheckError( 'Unsupported username format', f'Want "{site.regex_check}"' ), @@ -731,11 +709,11 @@ async def maigret( continue default_result: QueryResultWrapper = { 'site': site, - 'status': QueryResult( + 'status': MaigretCheckResult( username, sitename, '', - QueryStatus.UNKNOWN, + MaigretCheckStatus.UNKNOWN, error=CheckError('Request failed'), ), } @@ -819,8 +797,8 @@ async def site_self_check( } check_data = [ - (site.username_claimed, QueryStatus.CLAIMED), - (site.username_unclaimed, QueryStatus.AVAILABLE), + (site.username_claimed, MaigretCheckStatus.CLAIMED), + (site.username_unclaimed, MaigretCheckStatus.AVAILABLE), ] logger.info(f"Checking {site.name}...") @@ -859,7 +837,7 @@ async def site_self_check( site_status = result.status if site_status != status: - if site_status == QueryStatus.UNKNOWN: + if site_status == MaigretCheckStatus.UNKNOWN: msgs = site.absence_strs etype = site.check_type logger.warning( @@ -871,9 +849,9 @@ async def site_self_check( if skip_errors: pass # don't disable in case of available username - elif status == QueryStatus.CLAIMED: + elif status == MaigretCheckStatus.CLAIMED: changes["disabled"] = True - elif status == QueryStatus.CLAIMED: + elif status == MaigretCheckStatus.CLAIMED: logger.warning( f"Not found `{username}` in {site.name}, must be claimed" ) @@ -960,3 +938,38 @@ def disabled_count(lst): print(f"Unchecked sites verified: {unchecked_old_count - unchecked_new_count}") return total_disabled != 0 or unchecked_new_count != unchecked_old_count + + +def extract_ids_data(html_text, logger, site) -> Dict: + try: + return extract(html_text) + except Exception as e: + logger.warning(f"Error while parsing {site.name}: {e}", exc_info=True) + return {} + + +def parse_usernames(extracted_ids_data, logger) -> Dict: + new_usernames = {} + for k, v in extracted_ids_data.items(): + if "username" in k and not "usernames" in k: + new_usernames[v] = "username" + elif "usernames" in k: + try: + tree = ast.literal_eval(v) + if type(tree) == list: + for n in tree: + new_usernames[n] = "username" + except Exception as e: + logger.warning(e) + if k in SUPPORTED_IDS: + new_usernames[v] = k + return new_usernames + + +def update_results_info(results_info, extracted_ids_data, new_usernames): + results_info["ids_usernames"] = new_usernames + links = ascii_data_display(extracted_ids_data.get("links", "[]")) + if "website" in extracted_ids_data: + links.append(extracted_ids_data["website"]) + results_info["ids_links"] = links + return results_info diff --git a/maigret/errors.py b/maigret/errors.py index 295aa083..c9e1b07e 100644 --- a/maigret/errors.py +++ b/maigret/errors.py @@ -1,6 +1,6 @@ -from typing import Dict, List, Any +from typing import Dict, List, Any, Tuple -from .result import QueryResult +from .result import MaigretCheckResult from .types import QueryResultWrapper @@ -114,7 +114,7 @@ def extract_and_group(search_res: QueryResultWrapper) -> List[Dict[str, Any]]: errors_counts: Dict[str, int] = {} for r in search_res.values(): if r and isinstance(r, dict) and r.get('status'): - if not isinstance(r['status'], QueryResult): + if not isinstance(r['status'], MaigretCheckResult): continue err = r['status'].error @@ -133,3 +133,45 @@ def extract_and_group(search_res: QueryResultWrapper) -> List[Dict[str, Any]]: ) return counts + + +def notify_about_errors( + search_results: QueryResultWrapper, query_notify, show_statistics=False +) -> List[Tuple]: + """ + Prepare error notifications in search results, text + symbol, + to be displayed by notify object. + + Example: + [ + ("Too many errors of type "timeout" (50.0%)", "!") + ("Verbose error statistics:", "-") + ] + """ + results = [] + + errs = extract_and_group(search_results) + was_errs_displayed = False + for e in errs: + if not is_important(e): + continue + text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)' + solution = solution_of(e['err']) + if solution: + text = '. '.join([text, solution.capitalize()]) + + results.append((text, '!')) + was_errs_displayed = True + + if show_statistics: + results.append(('Verbose error statistics:', '-')) + for e in errs: + text = f'{e["err"]}: {round(e["perc"],2)}%' + results.append((text, '!')) + + if was_errs_displayed: + results.append( + ('You can see detailed site check errors with a flag `--print-errors`', '-') + ) + + return results \ No newline at end of file diff --git a/maigret/maigret.py b/maigret/maigret.py index 7a92c58b..1b639da8 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -45,34 +45,6 @@ from .permutator import Permute -def notify_about_errors( - search_results: QueryResultWrapper, query_notify, show_statistics=False -): - errs = errors.extract_and_group(search_results) - was_errs_displayed = False - for e in errs: - if not errors.is_important(e): - continue - text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)' - solution = errors.solution_of(e['err']) - if solution: - text = '. '.join([text, solution.capitalize()]) - - query_notify.warning(text, '!') - was_errs_displayed = True - - if show_statistics: - query_notify.warning(f'Verbose error statistics:') - for e in errs: - text = f'{e["err"]}: {round(e["perc"],2)}%' - query_notify.warning(text, '!') - - if was_errs_displayed: - query_notify.warning( - 'You can see detailed site check errors with a flag `--print-errors`' - ) - - def extract_ids_from_page(url, logger, timeout=5) -> dict: results = {} # url, headers @@ -693,7 +665,9 @@ async def main(): check_domains=args.with_domains, ) - notify_about_errors(results, query_notify, show_statistics=args.verbose) + errs = errors.notify_about_errors(results, query_notify, show_statistics=args.verbose) + for e in errs: + query_notify.warning(*e) if args.reports_sorting == "data": results = sort_report_by_data_points(results) diff --git a/maigret/notify.py b/maigret/notify.py index 056a1acd..c8012224 100644 --- a/maigret/notify.py +++ b/maigret/notify.py @@ -8,7 +8,7 @@ from colorama import Fore, Style, init -from .result import QueryStatus +from .result import MaigretCheckStatus from .utils import get_dict_ascii_tree @@ -245,7 +245,7 @@ def update(self, result, is_similar=False): ids_data_text = get_dict_ascii_tree(self.result.ids_data.items(), " ") # Output to the terminal is desired. - if result.status == QueryStatus.CLAIMED: + if result.status == MaigretCheckStatus.CLAIMED: color = Fore.BLUE if is_similar else Fore.GREEN status = "?" if is_similar else "+" notify = self.make_terminal_notify( @@ -255,7 +255,7 @@ def update(self, result, is_similar=False): color, result.site_url_user + ids_data_text, ) - elif result.status == QueryStatus.AVAILABLE: + elif result.status == MaigretCheckStatus.AVAILABLE: if not self.print_found_only: notify = self.make_terminal_notify( "-", @@ -264,7 +264,7 @@ def update(self, result, is_similar=False): Fore.YELLOW, "Not found!" + ids_data_text, ) - elif result.status == QueryStatus.UNKNOWN: + elif result.status == MaigretCheckStatus.UNKNOWN: if not self.skip_check_errors: notify = self.make_terminal_notify( "?", @@ -273,7 +273,7 @@ def update(self, result, is_similar=False): Fore.RED, str(self.result.error) + ids_data_text, ) - elif result.status == QueryStatus.ILLEGAL: + elif result.status == MaigretCheckStatus.ILLEGAL: if not self.print_found_only: text = "Illegal Username Format For This Site!" notify = self.make_terminal_notify( diff --git a/maigret/report.py b/maigret/report.py index dedb7d7e..54a80130 100644 --- a/maigret/report.py +++ b/maigret/report.py @@ -13,7 +13,7 @@ from jinja2 import Template from .checking import SUPPORTED_IDS -from .result import QueryStatus +from .result import MaigretCheckStatus from .sites import MaigretDatabase from .utils import is_country_tag, CaseConverter, enrich_link_str @@ -142,7 +142,7 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase if not status: # FIXME: currently in case of timeout continue - if dictionary["status"].status != QueryStatus.CLAIMED: + if dictionary["status"].status != MaigretCheckStatus.CLAIMED: continue site_fallback_name = dictionary.get( @@ -341,7 +341,7 @@ def generate_report_context(username_results: list): new_ids.append((u, utype)) usernames[u] = {"type": utype} - if status.status == QueryStatus.CLAIMED: + if status.status == MaigretCheckStatus.CLAIMED: found_accounts += 1 dictionary["found"] = True else: @@ -421,7 +421,7 @@ def generate_txt_report(username: str, results: dict, file): continue if ( dictionary.get("status") - and dictionary["status"].status == QueryStatus.CLAIMED + and dictionary["status"].status == MaigretCheckStatus.CLAIMED ): exists_counter += 1 file.write(dictionary["url_user"] + "\n") @@ -438,7 +438,7 @@ def generate_json_report(username: str, results: dict, file, report_type): if not site_result or not site_result.get("status"): continue - if site_result["status"].status != QueryStatus.CLAIMED: + if site_result["status"].status != MaigretCheckStatus.CLAIMED: continue data = dict(site_result) @@ -499,7 +499,7 @@ def design_xmind_sheet(sheet, username, results): continue result_status = dictionary.get("status") # TODO: fix the reason - if not result_status or result_status.status != QueryStatus.CLAIMED: + if not result_status or result_status.status != MaigretCheckStatus.CLAIMED: continue stripped_tags = list(map(lambda x: x.strip(), result_status.tags)) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index aa22eafd..45a468ef 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -17376,7 +17376,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzM0NDE4ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiYzRlNDQ4ZTgtZmFmNC00OWY1LTkyYmMtZWVmZWMzNWNlOTM1In0.nm4mnYvn8hm3u5gfNXh1r451U-R5O2MFOqz40DqixQo" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzM2MTc5MjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiNGYxM2M4N2ItYWMwMy00Y2JhLWExMDctNmNiODhmM2U3NjZjIn0.Y7CWEWckdSMsmJ8ROPmhHR6el2QCYJRDl0RLPpdJOKc" }, "activation": { "url": "https://vimeo.com/_rv/viewer", diff --git a/maigret/result.py b/maigret/result.py index 7eb59e3f..3258a1af 100644 --- a/maigret/result.py +++ b/maigret/result.py @@ -6,7 +6,7 @@ from enum import Enum -class QueryStatus(Enum): +class MaigretCheckStatus(Enum): """Query Status Enumeration. Describes status of query about a given username. @@ -29,10 +29,9 @@ def __str__(self): return self.value -class QueryResult: - """Query Result Object. - - Describes result of query about a given username. +class MaigretCheckResult: + """ + Describes result of checking a given username on a given site """ def __init__( @@ -47,11 +46,7 @@ def __init__( error=None, tags=[], ): - """Create Query Result Object. - - Contains information about a specific method of detecting usernames on - a given type of web sites. - + """ Keyword Arguments: self -- This object. username -- String indicating username that query result @@ -98,7 +93,7 @@ def json(self): } def is_found(self): - return self.status == QueryStatus.CLAIMED + return self.status == MaigretCheckStatus.CLAIMED def __str__(self): """Convert Object To String. diff --git a/maigret/submit.py b/maigret/submit.py index 33d10909..3092e352 100644 --- a/maigret/submit.py +++ b/maigret/submit.py @@ -9,11 +9,12 @@ from colorama import Fore, Style from .activation import import_aiohttp_cookies -from .result import QueryResult +from .result import MaigretCheckResult from .settings import Settings from .sites import MaigretDatabase, MaigretEngine, MaigretSite from .utils import get_random_user_agent - +from .checking import site_self_check +from .utils import get_match_ratio class CloudflareSession: @@ -73,6 +74,9 @@ async def close(self): @staticmethod def get_alexa_rank(site_url_main): + import requests + import xml.etree.ElementTree as ElementTree + url = f"http://data.alexa.com/data?cli=10&url={site_url_main}" xml_data = requests.get(url).text root = ElementTree.fromstring(xml_data) @@ -91,7 +95,7 @@ def extract_mainpage_url(url): async def site_self_check(self, site, semaphore, silent=False): # Call the general function from the checking.py - changes = await checking_site_self_check( + changes = await site_self_check( site=site, logger=self.logger, semaphore=semaphore, diff --git a/tests/conftest.py b/tests/conftest.py index 55eb3471..cbef55d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,26 @@ empty_mark = Mark('', (), {}) +RESULTS_EXAMPLE = { + 'Reddit': { + 'cookies': None, + 'parsing_enabled': False, + 'url_main': 'https://www.reddit.com/', + 'username': 'Skyeng', + }, + 'GooglePlayStore': { + 'cookies': None, + 'http_status': 200, + 'is_similar': False, + 'parsing_enabled': False, + 'rank': 1, + 'url_main': 'https://play.google.com/store', + 'url_user': 'https://play.google.com/store/apps/developer?id=Skyeng', + 'username': 'Skyeng', + }, +} + + def by_slow_marker(item): return item.get_closest_marker('slow', default=empty_mark).name diff --git a/tests/test_maigret.py b/tests/test_maigret.py index 9212d70e..d9d9fc97 100644 --- a/tests/test_maigret.py +++ b/tests/test_maigret.py @@ -12,28 +12,8 @@ extract_ids_from_results, ) from maigret.sites import MaigretSite -from maigret.result import QueryResult, QueryStatus - - -RESULTS_EXAMPLE = { - 'Reddit': { - 'cookies': None, - 'parsing_enabled': False, - 'url_main': 'https://www.reddit.com/', - 'username': 'Skyeng', - }, - 'GooglePlayStore': { - 'cookies': None, - 'http_status': 200, - 'is_similar': False, - 'parsing_enabled': False, - 'rank': 1, - 'url_main': 'https://play.google.com/store', - 'url_user': 'https://play.google.com/store/apps/developer?id=Skyeng', - 'username': 'Skyeng', - }, -} - +from maigret.result import MaigretCheckResult, MaigretCheckStatus +from tests.conftest import RESULTS_EXAMPLE @pytest.mark.slow @pytest.mark.asyncio @@ -86,12 +66,12 @@ def test_maigret_results(test_db): del results['GooglePlayStore']['site'] reddit_status = results['Reddit']['status'] - assert isinstance(reddit_status, QueryResult) - assert reddit_status.status == QueryStatus.ILLEGAL + assert isinstance(reddit_status, MaigretCheckResult) + assert reddit_status.status == MaigretCheckStatus.ILLEGAL playstore_status = results['GooglePlayStore']['status'] - assert isinstance(playstore_status, QueryResult) - assert playstore_status.status == QueryStatus.CLAIMED + assert isinstance(playstore_status, MaigretCheckResult) + assert playstore_status.status == MaigretCheckStatus.CLAIMED del results['Reddit']['status'] del results['GooglePlayStore']['status'] diff --git a/tests/test_notify.py b/tests/test_notify.py index d5f21baa..4c1de706 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -1,6 +1,6 @@ from maigret.errors import CheckError from maigret.notify import QueryNotifyPrint -from maigret.result import QueryStatus, QueryResult +from maigret.result import MaigretCheckStatus, MaigretCheckResult def test_notify_illegal(): @@ -8,9 +8,9 @@ def test_notify_illegal(): assert ( n.update( - QueryResult( + MaigretCheckResult( username="test", - status=QueryStatus.ILLEGAL, + status=MaigretCheckStatus.ILLEGAL, site_name="TEST_SITE", site_url_user="http://example.com/test", ) @@ -24,9 +24,9 @@ def test_notify_claimed(): assert ( n.update( - QueryResult( + MaigretCheckResult( username="test", - status=QueryStatus.CLAIMED, + status=MaigretCheckStatus.CLAIMED, site_name="TEST_SITE", site_url_user="http://example.com/test", ) @@ -40,9 +40,9 @@ def test_notify_available(): assert ( n.update( - QueryResult( + MaigretCheckResult( username="test", - status=QueryStatus.AVAILABLE, + status=MaigretCheckStatus.AVAILABLE, site_name="TEST_SITE", site_url_user="http://example.com/test", ) @@ -53,9 +53,9 @@ def test_notify_available(): def test_notify_unknown(): n = QueryNotifyPrint(color=False) - result = QueryResult( + result = MaigretCheckResult( username="test", - status=QueryStatus.UNKNOWN, + status=MaigretCheckStatus.UNKNOWN, site_name="TEST_SITE", site_url_user="http://example.com/test", ) diff --git a/tests/test_report.py b/tests/test_report.py index 9a5ed3b5..47c00a03 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -20,12 +20,12 @@ generate_json_report, get_plaintext_report, ) -from maigret.result import QueryResult, QueryStatus +from maigret.result import MaigretCheckResult, MaigretCheckStatus from maigret.sites import MaigretSite -GOOD_RESULT = QueryResult('', '', '', QueryStatus.CLAIMED) -BAD_RESULT = QueryResult('', '', '', QueryStatus.AVAILABLE) +GOOD_RESULT = MaigretCheckResult('', '', '', MaigretCheckStatus.CLAIMED) +BAD_RESULT = MaigretCheckResult('', '', '', MaigretCheckStatus.AVAILABLE) EXAMPLE_RESULTS = { 'GitHub': { @@ -33,11 +33,11 @@ 'parsing_enabled': True, 'url_main': 'https://www.github.com/', 'url_user': 'https://www.github.com/test', - 'status': QueryResult( + 'status': MaigretCheckResult( 'test', 'GitHub', 'https://www.github.com/test', - QueryStatus.CLAIMED, + MaigretCheckStatus.CLAIMED, tags=['test_tag'], ), 'http_status': 200, diff --git a/utils/import_sites.py b/utils/import_sites.py index bc49d26e..6b047b37 100755 --- a/utils/import_sites.py +++ b/utils/import_sites.py @@ -8,7 +8,7 @@ import requests from maigret.maigret import * -from maigret.result import QueryStatus +from maigret.result import MaigretCheckStatus from maigret.sites import MaigretSite URL_RE = re.compile(r"https?://(www\.)?") @@ -31,7 +31,7 @@ async def maigret_check(site, site_data, username, status, logger): ) if results[site]['status'].status != status: - if results[site]['status'].status == QueryStatus.UNKNOWN: + if results[site]['status'].status == MaigretCheckStatus.UNKNOWN: msg = site_data.absence_strs etype = site_data.check_type context = results[site]['status'].context @@ -41,7 +41,7 @@ async def maigret_check(site, site_data, username, status, logger): # continue return False - if status == QueryStatus.CLAIMED: + if status == MaigretCheckStatus.CLAIMED: logger.debug(f'Not found {username} in {site}, must be claimed') logger.debug(results[site]) pass @@ -62,7 +62,7 @@ async def check_and_add_maigret_site(site_data, semaphore, logger, ok_usernames, for ok_username in ok_usernames: site_data.username_claimed = ok_username - status = QueryStatus.CLAIMED + status = MaigretCheckStatus.CLAIMED if await maigret_check(sitename, site_data, ok_username, status, logger): # print(f'{sitename} positive case is okay') positive = True @@ -70,7 +70,7 @@ async def check_and_add_maigret_site(site_data, semaphore, logger, ok_usernames, for bad_username in bad_usernames: site_data.username_unclaimed = bad_username - status = QueryStatus.AVAILABLE + status = MaigretCheckStatus.AVAILABLE if await maigret_check(sitename, site_data, bad_username, status, logger): # print(f'{sitename} negative case is okay') negative = True From aca05c4033cafba3faa2e28007b47e0da7d15a57 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 01:50:05 +0100 Subject: [PATCH 2/9] Coverage dep added --- poetry.lock | 128 ++++++++++++++++++++++++------------------------- pyproject.toml | 1 + sites.md | 15 +++--- 3 files changed, 73 insertions(+), 71 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3b8ddf19..8dd1d7fb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -576,73 +576,73 @@ files = [ [[package]] name = "coverage" -version = "7.6.8" +version = "7.6.9" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" files = [ - {file = "coverage-7.6.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b39e6011cd06822eb964d038d5dff5da5d98652b81f5ecd439277b32361a3a50"}, - {file = "coverage-7.6.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:63c19702db10ad79151a059d2d6336fe0c470f2e18d0d4d1a57f7f9713875dcf"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3985b9be361d8fb6b2d1adc9924d01dec575a1d7453a14cccd73225cb79243ee"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:644ec81edec0f4ad17d51c838a7d01e42811054543b76d4ba2c5d6af741ce2a6"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f188a2402f8359cf0c4b1fe89eea40dc13b52e7b4fd4812450da9fcd210181d"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e19122296822deafce89a0c5e8685704c067ae65d45e79718c92df7b3ec3d331"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13618bed0c38acc418896005732e565b317aa9e98d855a0e9f211a7ffc2d6638"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:193e3bffca48ad74b8c764fb4492dd875038a2f9925530cb094db92bb5e47bed"}, - {file = "coverage-7.6.8-cp310-cp310-win32.whl", hash = "sha256:3988665ee376abce49613701336544041f2117de7b7fbfe91b93d8ff8b151c8e"}, - {file = "coverage-7.6.8-cp310-cp310-win_amd64.whl", hash = "sha256:f56f49b2553d7dd85fd86e029515a221e5c1f8cb3d9c38b470bc38bde7b8445a"}, - {file = "coverage-7.6.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:86cffe9c6dfcfe22e28027069725c7f57f4b868a3f86e81d1c62462764dc46d4"}, - {file = "coverage-7.6.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d82ab6816c3277dc962cfcdc85b1efa0e5f50fb2c449432deaf2398a2928ab94"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13690e923a3932e4fad4c0ebfb9cb5988e03d9dcb4c5150b5fcbf58fd8bddfc4"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4be32da0c3827ac9132bb488d331cb32e8d9638dd41a0557c5569d57cf22c9c1"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44e6c85bbdc809383b509d732b06419fb4544dca29ebe18480379633623baafb"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:768939f7c4353c0fac2f7c37897e10b1414b571fd85dd9fc49e6a87e37a2e0d8"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e44961e36cb13c495806d4cac67640ac2866cb99044e210895b506c26ee63d3a"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ea8bb1ab9558374c0ab591783808511d135a833c3ca64a18ec927f20c4030f0"}, - {file = "coverage-7.6.8-cp311-cp311-win32.whl", hash = "sha256:629a1ba2115dce8bf75a5cce9f2486ae483cb89c0145795603d6554bdc83e801"}, - {file = "coverage-7.6.8-cp311-cp311-win_amd64.whl", hash = "sha256:fb9fc32399dca861584d96eccd6c980b69bbcd7c228d06fb74fe53e007aa8ef9"}, - {file = "coverage-7.6.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e683e6ecc587643f8cde8f5da6768e9d165cd31edf39ee90ed7034f9ca0eefee"}, - {file = "coverage-7.6.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1defe91d41ce1bd44b40fabf071e6a01a5aa14de4a31b986aa9dfd1b3e3e414a"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7ad66e8e50225ebf4236368cc43c37f59d5e6728f15f6e258c8639fa0dd8e6d"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fe47da3e4fda5f1abb5709c156eca207eacf8007304ce3019eb001e7a7204cb"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202a2d645c5a46b84992f55b0a3affe4f0ba6b4c611abec32ee88358db4bb649"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4674f0daa1823c295845b6a740d98a840d7a1c11df00d1fd62614545c1583787"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:74610105ebd6f33d7c10f8907afed696e79c59e3043c5f20eaa3a46fddf33b4c"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37cda8712145917105e07aab96388ae76e787270ec04bcb9d5cc786d7cbb8443"}, - {file = "coverage-7.6.8-cp312-cp312-win32.whl", hash = "sha256:9e89d5c8509fbd6c03d0dd1972925b22f50db0792ce06324ba069f10787429ad"}, - {file = "coverage-7.6.8-cp312-cp312-win_amd64.whl", hash = "sha256:379c111d3558272a2cae3d8e57e6b6e6f4fe652905692d54bad5ea0ca37c5ad4"}, - {file = "coverage-7.6.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b0c69f4f724c64dfbfe79f5dfb503b42fe6127b8d479b2677f2b227478db2eb"}, - {file = "coverage-7.6.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c15b32a7aca8038ed7644f854bf17b663bc38e1671b5d6f43f9a2b2bd0c46f63"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63068a11171e4276f6ece913bde059e77c713b48c3a848814a6537f35afb8365"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f4548c5ead23ad13fb7a2c8ea541357474ec13c2b736feb02e19a3085fac002"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b4b4299dd0d2c67caaaf286d58aef5e75b125b95615dda4542561a5a566a1e3"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9ebfb2507751f7196995142f057d1324afdab56db1d9743aab7f50289abd022"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c1b4474beee02ede1eef86c25ad4600a424fe36cff01a6103cb4533c6bf0169e"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9fd2547e6decdbf985d579cf3fc78e4c1d662b9b0ff7cc7862baaab71c9cc5b"}, - {file = "coverage-7.6.8-cp313-cp313-win32.whl", hash = "sha256:8aae5aea53cbfe024919715eca696b1a3201886ce83790537d1c3668459c7146"}, - {file = "coverage-7.6.8-cp313-cp313-win_amd64.whl", hash = "sha256:ae270e79f7e169ccfe23284ff5ea2d52a6f401dc01b337efb54b3783e2ce3f28"}, - {file = "coverage-7.6.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de38add67a0af869b0d79c525d3e4588ac1ffa92f39116dbe0ed9753f26eba7d"}, - {file = "coverage-7.6.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b07c25d52b1c16ce5de088046cd2432b30f9ad5e224ff17c8f496d9cb7d1d451"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62a66ff235e4c2e37ed3b6104d8b478d767ff73838d1222132a7a026aa548764"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09b9f848b28081e7b975a3626e9081574a7b9196cde26604540582da60235fdf"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:093896e530c38c8e9c996901858ac63f3d4171268db2c9c8b373a228f459bbc5"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a7b8ac36fd688c8361cbc7bf1cb5866977ece6e0b17c34aa0df58bda4fa18a4"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:38c51297b35b3ed91670e1e4efb702b790002e3245a28c76e627478aa3c10d83"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2e4e0f60cb4bd7396108823548e82fdab72d4d8a65e58e2c19bbbc2f1e2bfa4b"}, - {file = "coverage-7.6.8-cp313-cp313t-win32.whl", hash = "sha256:6535d996f6537ecb298b4e287a855f37deaf64ff007162ec0afb9ab8ba3b8b71"}, - {file = "coverage-7.6.8-cp313-cp313t-win_amd64.whl", hash = "sha256:c79c0685f142ca53256722a384540832420dff4ab15fec1863d7e5bc8691bdcc"}, - {file = "coverage-7.6.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ac47fa29d8d41059ea3df65bd3ade92f97ee4910ed638e87075b8e8ce69599e"}, - {file = "coverage-7.6.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:24eda3a24a38157eee639ca9afe45eefa8d2420d49468819ac5f88b10de84f4c"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4c81ed2820b9023a9a90717020315e63b17b18c274a332e3b6437d7ff70abe0"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd55f8fc8fa494958772a2a7302b0354ab16e0b9272b3c3d83cdb5bec5bd1779"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f39e2f3530ed1626c66e7493be7a8423b023ca852aacdc91fb30162c350d2a92"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:716a78a342679cd1177bc8c2fe957e0ab91405bd43a17094324845200b2fddf4"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:177f01eeaa3aee4a5ffb0d1439c5952b53d5010f86e9d2667963e632e30082cc"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:912e95017ff51dc3d7b6e2be158dedc889d9a5cc3382445589ce554f1a34c0ea"}, - {file = "coverage-7.6.8-cp39-cp39-win32.whl", hash = "sha256:4db3ed6a907b555e57cc2e6f14dc3a4c2458cdad8919e40b5357ab9b6db6c43e"}, - {file = "coverage-7.6.8-cp39-cp39-win_amd64.whl", hash = "sha256:428ac484592f780e8cd7b6b14eb568f7c85460c92e2a37cb0c0e5186e1a0d076"}, - {file = "coverage-7.6.8-pp39.pp310-none-any.whl", hash = "sha256:5c52a036535d12590c32c49209e79cabaad9f9ad8aa4cbd875b68c4d67a9cbce"}, - {file = "coverage-7.6.8.tar.gz", hash = "sha256:8b2b8503edb06822c86d82fa64a4a5cb0760bb8f31f26e138ec743f422f37cfc"}, + {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, + {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073"}, + {file = "coverage-7.6.9-cp310-cp310-win32.whl", hash = "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198"}, + {file = "coverage-7.6.9-cp310-cp310-win_amd64.whl", hash = "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717"}, + {file = "coverage-7.6.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9"}, + {file = "coverage-7.6.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3"}, + {file = "coverage-7.6.9-cp311-cp311-win32.whl", hash = "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0"}, + {file = "coverage-7.6.9-cp311-cp311-win_amd64.whl", hash = "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b"}, + {file = "coverage-7.6.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8"}, + {file = "coverage-7.6.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f"}, + {file = "coverage-7.6.9-cp312-cp312-win32.whl", hash = "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692"}, + {file = "coverage-7.6.9-cp312-cp312-win_amd64.whl", hash = "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97"}, + {file = "coverage-7.6.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664"}, + {file = "coverage-7.6.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb"}, + {file = "coverage-7.6.9-cp313-cp313-win32.whl", hash = "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba"}, + {file = "coverage-7.6.9-cp313-cp313-win_amd64.whl", hash = "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1"}, + {file = "coverage-7.6.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419"}, + {file = "coverage-7.6.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9"}, + {file = "coverage-7.6.9-cp313-cp313t-win32.whl", hash = "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b"}, + {file = "coverage-7.6.9-cp313-cp313t-win_amd64.whl", hash = "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611"}, + {file = "coverage-7.6.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:adb697c0bd35100dc690de83154627fbab1f4f3c0386df266dded865fc50a902"}, + {file = "coverage-7.6.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:be57b6d56e49c2739cdf776839a92330e933dd5e5d929966fbbd380c77f060be"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1592791f8204ae9166de22ba7e6705fa4ebd02936c09436a1bb85aabca3e599"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e12ae8cc979cf83d258acb5e1f1cf2f3f83524d1564a49d20b8bec14b637f08"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb5555cff66c4d3d6213a296b360f9e1a8e323e74e0426b6c10ed7f4d021e464"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b9389a429e0e5142e69d5bf4a435dd688c14478a19bb901735cdf75e57b13845"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:592ac539812e9b46046620341498caf09ca21023c41c893e1eb9dbda00a70cbf"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a27801adef24cc30871da98a105f77995e13a25a505a0161911f6aafbd66e678"}, + {file = "coverage-7.6.9-cp39-cp39-win32.whl", hash = "sha256:8e3c3e38930cfb729cb8137d7f055e5a473ddaf1217966aa6238c88bd9fd50e6"}, + {file = "coverage-7.6.9-cp39-cp39-win_amd64.whl", hash = "sha256:e28bf44afa2b187cc9f41749138a64435bf340adfcacb5b2290c070ce99839d4"}, + {file = "coverage-7.6.9-pp39.pp310-none-any.whl", hash = "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b"}, + {file = "coverage-7.6.9.tar.gz", hash = "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d"}, ] [package.dependencies] @@ -2939,4 +2939,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "7e36b57d14f5feedd75778934df5a24c669fd7dd3d5f0147f566ac4ea6eb1d27" +content-hash = "64a1fba0826fcf13840f573b9cd2754a26b337c3d7e5e195195d2a53d99db870" diff --git a/pyproject.toml b/pyproject.toml index 0fa84218..9aabe56b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ reportlab = "^4.2.0" mypy = "^1.13.0" tuna = "^0.5.11" black = "^24.10.0" +coverage = "^7.6.9" [tool.poetry.scripts] # Run with: poetry run maigret diff --git a/sites.md b/sites.md index 2c7cec46..b4f72960 100644 --- a/sites.md +++ b/sites.md @@ -1,5 +1,5 @@ -## List of supported sites (search methods): total 3126 +## List of supported sites (search methods): total 3127 Rank data fetched from Alexa by domains. @@ -3129,17 +3129,18 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.tnaflix.com) [www.tnaflix.com (https://www.tnaflix.com)](https://www.tnaflix.com)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://massagerepublic.com) [massagerepublic.com (https://massagerepublic.com)](https://massagerepublic.com)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://mynickname.com) [mynickname.com (https://mynickname.com)](https://mynickname.com)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain=https://substack.com) [Substack (https://substack.com)](https://substack.com)*: top 100M, blog* -The list was updated at (2024-12-06) +The list was updated at (2024-12-08) ## Statistics -Enabled/total sites: 2689/3126 = 86.02% +Enabled/total sites: 2690/3127 = 86.02% -Incomplete message checks: 404/2689 = 15.02% (false positive risks) +Incomplete message checks: 404/2690 = 15.02% (false positive risks) -Status code checks: 720/2689 = 26.78% (false positive risks) +Status code checks: 720/2690 = 26.77% (false positive risks) -False positive risk (total): 41.80% +False positive risk (total): 41.79% Top 20 profile URLs: - (796) `{urlMain}/index/8-0-{username} (uCoz)` @@ -3153,7 +3154,7 @@ Top 20 profile URLs: - (88) `/users/{username}` - (87) `{urlMain}/u/{username}/summary (Discourse)` - (54) `/wiki/User:{username}` -- (51) `/@{username}` +- (52) `/@{username}` - (42) `SUBDOMAIN` - (41) `/members/?username={username}` - (32) `/members/{username}` From 97068a46293b0145a25bbf2c6a2f8c007833ff77 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 01:51:58 +0100 Subject: [PATCH 3/9] Fix coverage launch in GH workflow --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e83b38ac..fc00d193 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,10 +29,10 @@ jobs: python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - coverage run -m pytest --reruns 3 --reruns-delay 5 + python -m coverage run -m pytest --reruns 3 --reruns-delay 5 # Fail if coverage is less than 60% - coverage report --fail-under=60 - coverage html + python -m coverage report --fail-under=60 + python -m coverage html - name: Upload coverage report uses: actions/upload-artifact@v3 with: From 8794b7072b69cbd72938afbdd85a44a42cf50cbe Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 01:54:49 +0100 Subject: [PATCH 4/9] Another coverage fix --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fc00d193..4be6ea5c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -25,7 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install poetry + python -m pip install poetry coverage python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | From 1522d870f83e904eeb4ef7f8dedd597ee39ed574 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 01:59:45 +0100 Subject: [PATCH 5/9] Added separate coverage test --- .github/workflows/coverage-test.yml | 31 ++++++++++++++++++++++++++++ .github/workflows/python-package.yml | 12 ++--------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/coverage-test.yml diff --git a/.github/workflows/coverage-test.yml b/.github/workflows/coverage-test.yml new file mode 100644 index 00000000..4bfec950 --- /dev/null +++ b/.github/workflows/coverage-test.yml @@ -0,0 +1,31 @@ +name: Linting and testing + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + python -m poetry install --with dev + - name: Test with Coverage and Pytest (Fail if coverage is low) + run: | + coverage run --source=./maigret -m pytest tests + # Fail if coverage is less than 60% + coverage report --fail-under=60 + coverage html + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: htmlcov + path: htmlcov \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4be6ea5c..c3276b6a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -25,16 +25,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install poetry coverage + python -m pip install poetry python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - python -m coverage run -m pytest --reruns 3 --reruns-delay 5 - # Fail if coverage is less than 60% - python -m coverage report --fail-under=60 - python -m coverage html - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: htmlcov - path: htmlcov \ No newline at end of file + poetry run pytest --reruns 3 --reruns-delay 5 From 60c2426a11c368ca746d31062997cc3e2e9cf34c Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 02:01:33 +0100 Subject: [PATCH 6/9] Update coverage module --- .github/workflows/coverage-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage-test.yml b/.github/workflows/coverage-test.yml index 4bfec950..0f87b571 100644 --- a/.github/workflows/coverage-test.yml +++ b/.github/workflows/coverage-test.yml @@ -1,4 +1,4 @@ -name: Linting and testing +name: Coverage Test (60% minimum) on: push: @@ -20,10 +20,10 @@ jobs: python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - coverage run --source=./maigret -m pytest tests + poetry run coverage run --source=./maigret -m pytest tests # Fail if coverage is less than 60% - coverage report --fail-under=60 - coverage html + poetry run coverage report --fail-under=60 + poetry run coverage html - name: Upload coverage report uses: actions/upload-artifact@v3 with: From eaaafec7116671a789a1fb60ee2ff3e45bbb3d24 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 02:04:04 +0100 Subject: [PATCH 7/9] Added reruns --- .github/workflows/coverage-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage-test.yml b/.github/workflows/coverage-test.yml index 0f87b571..42d01c67 100644 --- a/.github/workflows/coverage-test.yml +++ b/.github/workflows/coverage-test.yml @@ -20,7 +20,7 @@ jobs: python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - poetry run coverage run --source=./maigret -m pytest tests + poetry run coverage run --source=./maigret -m pytest --reruns 3 --reruns-delay 5 tests # Fail if coverage is less than 60% poetry run coverage report --fail-under=60 poetry run coverage html From e0fd08b021f00ca9c6dd3c3e127a23c13d0446d0 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 02:06:54 +0100 Subject: [PATCH 8/9] Coverage moved back to tests --- .github/workflows/coverage-test.yml | 31 ---------------------------- .github/workflows/python-package.yml | 9 +++++++- 2 files changed, 8 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/coverage-test.yml diff --git a/.github/workflows/coverage-test.yml b/.github/workflows/coverage-test.yml deleted file mode 100644 index 42d01c67..00000000 --- a/.github/workflows/coverage-test.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Coverage Test (60% minimum) - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - types: [opened, synchronize, reopened] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install poetry - python -m poetry install --with dev - - name: Test with Coverage and Pytest (Fail if coverage is low) - run: | - poetry run coverage run --source=./maigret -m pytest --reruns 3 --reruns-delay 5 tests - # Fail if coverage is less than 60% - poetry run coverage report --fail-under=60 - poetry run coverage html - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: htmlcov - path: htmlcov \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c3276b6a..99aaf660 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,4 +29,11 @@ jobs: python -m poetry install --with dev - name: Test with Coverage and Pytest (Fail if coverage is low) run: | - poetry run pytest --reruns 3 --reruns-delay 5 + poetry run coverage run --source=./maigret -m pytest --reruns 3 --reruns-delay 5 tests + poetry run coverage report --fail-under=58 + poetry run coverage html + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: htmlcov + path: htmlcov \ No newline at end of file From bd7154cf2630d05e31f2d074ab7d724924cdf32d Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 8 Dec 2024 02:10:38 +0100 Subject: [PATCH 9/9] Added 2 tests --- .github/workflows/python-package.yml | 2 +- tests/test_errors.py | 28 +++++++++++++++++++++++ tests/test_permutator.py | 34 ++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/test_errors.py create mode 100644 tests/test_permutator.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 99aaf660..4d0682ff 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,7 @@ jobs: - name: Test with Coverage and Pytest (Fail if coverage is low) run: | poetry run coverage run --source=./maigret -m pytest --reruns 3 --reruns-delay 5 tests - poetry run coverage report --fail-under=58 + poetry run coverage report --fail-under=60 poetry run coverage html - name: Upload coverage report uses: actions/upload-artifact@v3 diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 00000000..d2067cc7 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,28 @@ +import pytest +from maigret.errors import notify_about_errors, CheckError +from maigret.types import QueryResultWrapper +from maigret.result import MaigretCheckResult, MaigretCheckStatus + + +def test_notify_about_errors(): + results = { + 'site1': {'status': MaigretCheckResult('', '', '', MaigretCheckStatus.UNKNOWN, error=CheckError('Captcha'))}, + 'site2': {'status': MaigretCheckResult('', '', '', MaigretCheckStatus.UNKNOWN, error=CheckError('Bot protection'))}, + 'site3': {'status': MaigretCheckResult('', '', '', MaigretCheckStatus.UNKNOWN, error=CheckError('Access denied'))}, + 'site4': {'status': MaigretCheckResult('', '', '', MaigretCheckStatus.CLAIMED, error=None)}, + } + + results = notify_about_errors(results, query_notify=None, show_statistics=True) + + # Check the output + expected_output = [ + ('Too many errors of type "Captcha" (25.0%). Try to switch to another ip address or to use service cookies', '!'), + ('Too many errors of type "Bot protection" (25.0%). Try to switch to another ip address', '!'), + ('Too many errors of type "Access denied" (25.0%)', '!'), + ('Verbose error statistics:', '-'), + ('Captcha: 25.0%', '!'), + ('Bot protection: 25.0%', '!'), + ('Access denied: 25.0%', '!'), + ('You can see detailed site check errors with a flag `--print-errors`', '-'), + ] + assert results == expected_output \ No newline at end of file diff --git a/tests/test_permutator.py b/tests/test_permutator.py new file mode 100644 index 00000000..21a20a88 --- /dev/null +++ b/tests/test_permutator.py @@ -0,0 +1,34 @@ +import pytest +from maigret.permutator import Permute + + +def test_gather_strict(): + elements = {'a': 1, 'b': 2} + permute = Permute(elements) + result = permute.gather(method="strict") + expected = { + 'a_b': 1, 'b_a': 2, + 'a-b': 1, 'b-a': 2, + 'a.b': 1, 'b.a': 2, + 'ab': 1, 'ba': 2, + '_ab': 1, 'ab_': 1, + '_ba': 2, 'ba_': 2 + } + assert result == expected + + +def test_gather_all(): + elements = {'a': 1, 'b': 2} + permute = Permute(elements) + result = permute.gather(method="all") + expected = { + 'a': 1, '_a': 1, 'a_': 1, + 'b': 2, '_b': 2, 'b_': 2, + 'a_b': 1, 'b_a': 2, + 'a-b': 1, 'b-a': 2, + 'a.b': 1, 'b.a': 2, + 'ab': 1, 'ba': 2, + '_ab': 1, 'ab_': 1, + '_ba': 2, 'ba_': 2 + } + assert result == expected \ No newline at end of file