diff --git a/README.md b/README.md index e16ffd37..25279cc8 100644 --- a/README.md +++ b/README.md @@ -553,9 +553,9 @@ PROMPT KEYS: $ buku --random --print -44. Print out 3 **random** bookmarks **ordered** by title (reversed) and url: +44. Print out 3 **random** bookmarks **ordered** by netloc (reversed), title and url: - $ buku --random 3 --order ,-title,+url --print + $ buku --random 3 --order ,-netloc,title,+url --print 45. Print out a single **random** bookmark matching **search** criteria, and **export** into a Markdown file (in DB order): diff --git a/buku b/buku index d4a58949..41a8214f 100755 --- a/buku +++ b/buku @@ -51,12 +51,12 @@ from typing import Any, Dict, List, Optional, Tuple, NamedTuple from collections.abc import Sequence, Set, Callable from warnings import warn import xml.etree.ElementTree as ET +from urllib.parse import urlparse # urllib3.util.parse_url() encodes netloc import urllib3 from bs4 import BeautifulSoup from bs4.dammit import EncodingDetector -from urllib3.exceptions import LocationParseError -from urllib3.util import Retry, make_headers, parse_url +from urllib3.util import Retry, make_headers try: from mypy_extensions import TypedDict @@ -462,6 +462,10 @@ class BookmarkVar(NamedTuple): def taglist(self) -> List[str]: return [x for x in self.tags_raw.split(',') if x] + @property + def netloc(self) -> str: + return get_netloc(self.url) or '' + bookmark_vars = lambda xs: ((x if isinstance(x, BookmarkVar) else BookmarkVar(*x)) for x in xs) @@ -593,6 +597,7 @@ class BukuDb: # Create a connection conn = sqlite3.connect(dbfile, check_same_thread=False) conn.create_function('REGEXP', 2, regexp) + conn.create_function('NETLOC', 1, get_netloc) cur = conn.cursor() # Create table if it doesn't exist @@ -629,13 +634,13 @@ class BukuDb: Fields are listed in priority order, with '+'/'-' prefix signifying ASC/DESC; assuming ASC if not specified. Other than names from DB, you can pass those from JSON export.""" names = {'index': 'id', 'uri': 'url', 'description': 'desc', **({'title': 'metadata'} if for_db else {'metadata': 'title'})} - valid = list(names) + list(names.values()) + ['tags'] + valid = list(names) + list(names.values()) + ['tags', 'netloc'] _fields = [(re.sub(r'^[+-]', '', s), not s.startswith('-')) for s in (fields or [])] _fields = [(names.get(field, field), direction) for field, direction in _fields if field in valid] return _fields or [('id', True)] def _sort(self, records: List[BookmarkVar], fields=['+id'], ignore_case=True) -> List[BookmarkVar]: - text_fields = (set() if not ignore_case else {'url', 'desc', 'title', 'tags'}) + text_fields = (set() if not ignore_case else {'url', 'desc', 'title', 'tags', 'netloc'}) get = lambda x, k: (getattr(x, k) if k not in text_fields else str(getattr(x, k) or '').lower()) order = self._ordering(fields, for_db=False) return sorted(bookmark_vars(records), key=lambda x: [SortKey(get(x, k), ascending=asc) for k, asc in order]) @@ -643,8 +648,8 @@ class BukuDb: def _order(self, fields=['+id'], ignore_case=True) -> str: """Converts field list to SQL 'ORDER BY' parameters. (See also BukuDb._ordering().)""" text_fields = (set() if not ignore_case else {'url', 'desc', 'metadata', 'tags'}) - return ', '.join(f'{field if field not in text_fields else "LOWER("+field+")"} {"ASC" if direction else "DESC"}' - for field, direction in self._ordering(fields)) + get = lambda field: ('LOWER(NETLOC(url))' if field == 'netloc' else field if field not in text_fields else f'LOWER({field})') + return ', '.join(f'{get(field)} {"ASC" if direction else "DESC"}' for field, direction in self._ordering(fields)) def get_rec_all(self, *, lock: bool = True, order: List[str] = ['id']): """Get all the bookmarks in the database. @@ -4053,6 +4058,20 @@ def import_html(html_soup: BeautifulSoup, add_parent_folder_as_tag: bool, newtag ) +def get_netloc(url): + """Get the netloc token, or None.""" + + try: + netloc = urlparse(url).netloc + if not netloc and not urlparse(url).scheme: + # Try to prepend '//' and get netloc + netloc = urlparse('//' + url).netloc + return netloc or None + except Exception as e: + LOGERR('%s, URL: %s', e, url) + return None + + def is_bad_url(url): """Check if URL is malformed. @@ -4069,16 +4088,8 @@ def is_bad_url(url): True if URL is malformed, False otherwise. """ - # Get the netloc token - try: - netloc = parse_url(url).netloc - if not netloc: - # Try of prepend '//' and get netloc - netloc = parse_url('//' + url).netloc - if not netloc: - return True - except LocationParseError as e: - LOGERR('%s, URL: %s', e, url) + netloc = get_netloc(url) + if not netloc: return True LOGDBG('netloc: %s', netloc) @@ -4088,10 +4099,7 @@ def is_bad_url(url): return True # netloc should have at least one '.' - if netloc.rfind('.') < 0: - return True - - return False + return '.' not in netloc def is_nongeneric_url(url): @@ -4277,6 +4285,14 @@ def get_data_from_page(resp): return (None, None, None) +def extract_auth(url): + """Convert an url into an (auth, url) tuple [the returned URL will contain no auth part].""" + _url = urlparse(url) + if _url.username is None: # no '@' in netloc + return None, url + auth = _url.username + ('' if _url.password is None else f':{_url.password}') + return auth, url.replace(auth + '@', '') + def gen_headers(): """Generate headers for network connection.""" @@ -4293,15 +4309,14 @@ def gen_headers(): MYPROXY = os.environ.get('https_proxy') if MYPROXY: try: - url = parse_url(MYPROXY) + auth, MYPROXY = extract_auth(MYPROXY) except Exception as e: LOGERR(e) return # Strip username and password (if present) and update headers - if url.auth: - MYPROXY = MYPROXY.replace(url.auth + '@', '') - auth_headers = make_headers(basic_auth=url.auth) + if auth: + auth_headers = make_headers(basic_auth=auth) MYHEADERS.update(auth_headers) LOGDBG('proxy: [%s]', MYPROXY) @@ -5179,7 +5194,7 @@ def browse(url): If True, tries to open links in a GUI based browser. """ - if not parse_url(url).scheme: + if not urlparse(url).scheme: # Prefix with 'http://' if no scheme # Otherwise, opening in browser fails anyway # We expect http to https redirection diff --git a/buku.1 b/buku.1 index 13b9fa0c..48ec4a44 100644 --- a/buku.1 +++ b/buku.1 @@ -200,7 +200,7 @@ Exclude bookmarks matching the specified keywords. Works with --sany, --sall, -- Output random bookmarks out of the selection (1 unless amount is specified). .TP .BI \--order " fields [...]" -Order printed/exported records by the given fields (from DB or JSON). You can specify sort direction for each by prepending '+'/'-' (default is '+'). +Order printed/exported records by the given fields (from DB or JSON) and/or netloc. You can specify sort direction for each by prepending '+'/'-' (default is '+'). .SH ENCRYPTION OPTIONS .TP .BI \-l " " \--lock " [N]" @@ -946,11 +946,11 @@ Print out a single \fBrandom\fR bookmark: .EE .PP .IP 44. 4 -Print out 3 \fBrandom\fR bookmarks \fBordered\fR by title (reversed) and url: +Print out 3 \fBrandom\fR bookmarks \fBordered\fR by netloc (reversed), title and url: .PP .EX .IP -.B buku --random 3 --order ,-title,+url +.B buku --random 3 --order ,-netloc,title,+url .EE .PP .IP 45. 4 diff --git a/bukuserver/filters.py b/bukuserver/filters.py index a695e131..4108d477 100644 --- a/bukuserver/filters.py +++ b/bukuserver/filters.py @@ -122,7 +122,7 @@ def clean(self, value): class BookmarkOrderFilter(BaseFilter): DIR_LIST = [('asc', _l('natural')), ('desc', _l('reversed'))] - FIELDS = ['index', 'url', 'title', 'description', 'tags'] + FIELDS = ['index', 'url', 'netloc', 'title', 'description', 'tags'] def __init__(self, field, *args, **kwargs): self.field = field diff --git a/bukuserver/translations/de/LC_MESSAGES/messages.po b/bukuserver/translations/de/LC_MESSAGES/messages.po index a922f9b5..aa20773f 100644 --- a/bukuserver/translations/de/LC_MESSAGES/messages.po +++ b/bukuserver/translations/de/LC_MESSAGES/messages.po @@ -196,73 +196,73 @@ msgstr "" msgid "Index" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:221 +#: /home/lex/Work/buku/bukuserver/views.py:223 #, python-format msgid "url invalid: %(url)s" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:232 +#: /home/lex/Work/buku/bukuserver/views.py:234 msgid "Failed to create record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:245 -#: /home/lex/Work/buku/bukuserver/views.py:552 +#: /home/lex/Work/buku/bukuserver/views.py:247 +#: /home/lex/Work/buku/bukuserver/views.py:554 msgid "Failed to delete record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:260 +#: /home/lex/Work/buku/bukuserver/views.py:262 msgid "Invalid search mode combination" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:348 +#: /home/lex/Work/buku/bukuserver/views.py:350 msgid "netloc match" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:381 +#: /home/lex/Work/buku/bukuserver/views.py:383 msgid "contain" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:382 +#: /home/lex/Work/buku/bukuserver/views.py:384 msgid "not contain" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:383 +#: /home/lex/Work/buku/bukuserver/views.py:385 msgid "number equal" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:384 +#: /home/lex/Work/buku/bukuserver/views.py:386 msgid "number not equal" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:385 +#: /home/lex/Work/buku/bukuserver/views.py:387 msgid "number greater than" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:386 +#: /home/lex/Work/buku/bukuserver/views.py:388 msgid "number smaller than" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:412 -#: /home/lex/Work/buku/bukuserver/views.py:570 +#: /home/lex/Work/buku/bukuserver/views.py:414 +#: /home/lex/Work/buku/bukuserver/views.py:572 msgid "Failed to update record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:427 +#: /home/lex/Work/buku/bukuserver/views.py:429 msgid "" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:433 -#: /home/lex/Work/buku/bukuserver/views.py:470 +#: /home/lex/Work/buku/bukuserver/views.py:435 +#: /home/lex/Work/buku/bukuserver/views.py:472 msgctxt "tag" msgid "Name" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:433 +#: /home/lex/Work/buku/bukuserver/views.py:435 msgctxt "tag" msgid "Usage Count" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:533 +#: /home/lex/Work/buku/bukuserver/views.py:535 msgid "top most common" msgstr "" @@ -539,6 +539,9 @@ msgstr "" msgid "by url" msgstr "" +msgid "by netloc" +msgstr "" + msgid "by title" msgstr "" diff --git a/bukuserver/translations/fr/LC_MESSAGES/messages.po b/bukuserver/translations/fr/LC_MESSAGES/messages.po index ac6c8e78..2e82b5c3 100644 --- a/bukuserver/translations/fr/LC_MESSAGES/messages.po +++ b/bukuserver/translations/fr/LC_MESSAGES/messages.po @@ -196,73 +196,73 @@ msgstr "" msgid "Index" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:221 +#: /home/lex/Work/buku/bukuserver/views.py:223 #, python-format msgid "url invalid: %(url)s" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:232 +#: /home/lex/Work/buku/bukuserver/views.py:234 msgid "Failed to create record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:245 -#: /home/lex/Work/buku/bukuserver/views.py:552 +#: /home/lex/Work/buku/bukuserver/views.py:247 +#: /home/lex/Work/buku/bukuserver/views.py:554 msgid "Failed to delete record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:260 +#: /home/lex/Work/buku/bukuserver/views.py:262 msgid "Invalid search mode combination" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:348 +#: /home/lex/Work/buku/bukuserver/views.py:350 msgid "netloc match" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:381 +#: /home/lex/Work/buku/bukuserver/views.py:383 msgid "contain" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:382 +#: /home/lex/Work/buku/bukuserver/views.py:384 msgid "not contain" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:383 +#: /home/lex/Work/buku/bukuserver/views.py:385 msgid "number equal" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:384 +#: /home/lex/Work/buku/bukuserver/views.py:386 msgid "number not equal" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:385 +#: /home/lex/Work/buku/bukuserver/views.py:387 msgid "number greater than" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:386 +#: /home/lex/Work/buku/bukuserver/views.py:388 msgid "number smaller than" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:412 -#: /home/lex/Work/buku/bukuserver/views.py:570 +#: /home/lex/Work/buku/bukuserver/views.py:414 +#: /home/lex/Work/buku/bukuserver/views.py:572 msgid "Failed to update record." msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:427 +#: /home/lex/Work/buku/bukuserver/views.py:429 msgid "" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:433 -#: /home/lex/Work/buku/bukuserver/views.py:470 +#: /home/lex/Work/buku/bukuserver/views.py:435 +#: /home/lex/Work/buku/bukuserver/views.py:472 msgctxt "tag" msgid "Name" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:433 +#: /home/lex/Work/buku/bukuserver/views.py:435 msgctxt "tag" msgid "Usage Count" msgstr "" -#: /home/lex/Work/buku/bukuserver/views.py:533 +#: /home/lex/Work/buku/bukuserver/views.py:535 msgid "top most common" msgstr "" @@ -539,6 +539,9 @@ msgstr "" msgid "by url" msgstr "" +msgid "by netloc" +msgstr "" + msgid "by title" msgstr "" diff --git a/bukuserver/translations/messages_custom.pot b/bukuserver/translations/messages_custom.pot index 2f8e80d4..7c3d63bc 100644 --- a/bukuserver/translations/messages_custom.pot +++ b/bukuserver/translations/messages_custom.pot @@ -10,6 +10,9 @@ msgstr "" msgid "by url" msgstr "" +msgid "by netloc" +msgstr "" + msgid "by title" msgstr "" diff --git a/bukuserver/translations/ru/LC_MESSAGES/messages.mo b/bukuserver/translations/ru/LC_MESSAGES/messages.mo index f909f5aa..18548136 100644 Binary files a/bukuserver/translations/ru/LC_MESSAGES/messages.mo and b/bukuserver/translations/ru/LC_MESSAGES/messages.mo differ diff --git a/bukuserver/translations/ru/LC_MESSAGES/messages.po b/bukuserver/translations/ru/LC_MESSAGES/messages.po index c3c2f1fc..1fb13c66 100644 --- a/bukuserver/translations/ru/LC_MESSAGES/messages.po +++ b/bukuserver/translations/ru/LC_MESSAGES/messages.po @@ -196,73 +196,73 @@ msgstr "Закладка" msgid "Index" msgstr "Номер" -#: /home/lex/Work/buku/bukuserver/views.py:221 +#: /home/lex/Work/buku/bukuserver/views.py:223 #, python-format msgid "url invalid: %(url)s" msgstr "некорректная ссылка: %(url)s" -#: /home/lex/Work/buku/bukuserver/views.py:232 +#: /home/lex/Work/buku/bukuserver/views.py:234 msgid "Failed to create record." msgstr "Ошибка создания записи." -#: /home/lex/Work/buku/bukuserver/views.py:245 -#: /home/lex/Work/buku/bukuserver/views.py:552 +#: /home/lex/Work/buku/bukuserver/views.py:247 +#: /home/lex/Work/buku/bukuserver/views.py:554 msgid "Failed to delete record." msgstr "Ошибка удаления записи." -#: /home/lex/Work/buku/bukuserver/views.py:260 +#: /home/lex/Work/buku/bukuserver/views.py:262 msgid "Invalid search mode combination" msgstr "Некорректная комбинация фильтров поиска" -#: /home/lex/Work/buku/bukuserver/views.py:348 +#: /home/lex/Work/buku/bukuserver/views.py:350 msgid "netloc match" msgstr "на сайт" -#: /home/lex/Work/buku/bukuserver/views.py:381 +#: /home/lex/Work/buku/bukuserver/views.py:383 msgid "contain" msgstr "содержат" -#: /home/lex/Work/buku/bukuserver/views.py:382 +#: /home/lex/Work/buku/bukuserver/views.py:384 msgid "not contain" msgstr "не содержат" -#: /home/lex/Work/buku/bukuserver/views.py:383 +#: /home/lex/Work/buku/bukuserver/views.py:385 msgid "number equal" msgstr "количество равно" -#: /home/lex/Work/buku/bukuserver/views.py:384 +#: /home/lex/Work/buku/bukuserver/views.py:386 msgid "number not equal" msgstr "количество не равно" -#: /home/lex/Work/buku/bukuserver/views.py:385 +#: /home/lex/Work/buku/bukuserver/views.py:387 msgid "number greater than" msgstr "количество больше чем" -#: /home/lex/Work/buku/bukuserver/views.py:386 +#: /home/lex/Work/buku/bukuserver/views.py:388 msgid "number smaller than" msgstr "количество меньше чем" -#: /home/lex/Work/buku/bukuserver/views.py:412 -#: /home/lex/Work/buku/bukuserver/views.py:570 +#: /home/lex/Work/buku/bukuserver/views.py:414 +#: /home/lex/Work/buku/bukuserver/views.py:572 msgid "Failed to update record." msgstr "Ошибка обновления записи." -#: /home/lex/Work/buku/bukuserver/views.py:427 +#: /home/lex/Work/buku/bukuserver/views.py:429 msgid "" msgstr "<БЕЗ ТЕГОВ>" -#: /home/lex/Work/buku/bukuserver/views.py:433 -#: /home/lex/Work/buku/bukuserver/views.py:470 +#: /home/lex/Work/buku/bukuserver/views.py:435 +#: /home/lex/Work/buku/bukuserver/views.py:472 msgctxt "tag" msgid "Name" msgstr "Тег" -#: /home/lex/Work/buku/bukuserver/views.py:433 +#: /home/lex/Work/buku/bukuserver/views.py:435 msgctxt "tag" msgid "Usage Count" msgstr "Число закладок" -#: /home/lex/Work/buku/bukuserver/views.py:533 +#: /home/lex/Work/buku/bukuserver/views.py:535 msgid "top most common" msgstr "самое распространённое" @@ -539,6 +539,9 @@ msgstr "по номеру" msgid "by url" msgstr "по ссылке" +msgid "by netloc" +msgstr "по сайту" + msgid "by title" msgstr "по названию" diff --git a/bukuserver/views.py b/bukuserver/views.py index 3c9c1eea..0c00eed9 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -117,7 +117,7 @@ def _create_ajax_loader(self, name, options): def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: LOG.debug("context: %s, name: %s", context, name) parsed_url = urlparse(model.url) - netloc = parsed_url.netloc + netloc = buku.get_netloc(model.url) or '' get_index_view_url = functools.partial(url_for, "bookmark.index_view") res = [] if netloc and not app_param('DISABLE_FAVICON'): @@ -156,7 +156,7 @@ def get_detail_value(self, context, model, name): for s in (value or '').split(',') if s.strip()) return Markup(f'
{"".join(tags)}
') if name == 'url': - res, netloc, scheme = [], (parsed := urlparse(value)).netloc, parsed.scheme + res, netloc, scheme = [], buku.get_netloc(value), urlparse(value).scheme if netloc and not app_param('DISABLE_FAVICON', False): icon = f'' res += [link(icon, url_for('bookmark.index_view', flt0_url_netloc_match=netloc), html=True)] @@ -293,7 +293,7 @@ def get_one(self, id): bm_sns = types.SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) for field in list(BookmarkField): setattr(bm_sns, field.name.lower(), format_value(field, bookmark, spacing=' ')) - session['netloc'] = urlparse(bookmark.url).netloc + session['netloc'] = buku.get_netloc(bookmark.url) or '' return bm_sns def get_pk_value(self, model): @@ -344,7 +344,7 @@ def scaffold_filters(self, name): elif name == BookmarkField.URL.name.lower(): def netloc_match_func(query, value, index): - return filter(lambda x: urlparse(x[index]).netloc == value, query) + return filter(lambda x: (buku.get_netloc(x[index]) or '') == value, query) res += [ bs_filters.BookmarkBaseFilter(name, _l('netloc match'), netloc_match_func), @@ -593,7 +593,7 @@ def index(self): data = StatisticView._data if not data or request.method == 'POST': all_bookmarks = self.bukudb.get_rec_all() - netlocs = [urlparse(x.url).netloc for x in all_bookmarks] + netlocs = [buku.get_netloc(x.url) or '' for x in all_bookmarks] tags = [s for x in all_bookmarks for s in x.taglist] titles = [x.title for x in all_bookmarks] data = StatisticView._data = { diff --git a/tests/test_buku.py b/tests/test_buku.py index 5ab85670..e5badc73 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -13,7 +13,7 @@ import pytest from buku import DELIM, FIELD_FILTER, ALL_FIELDS, SortKey, FetchResult, is_int, prep_tag_search, \ - print_rec_with_filter, parse_range, split_by_marker + print_rec_with_filter, get_netloc, extract_auth, parse_range, split_by_marker def check_import_html_results_contains(result, expected_result): @@ -26,20 +26,58 @@ def check_import_html_results_contains(result, expected_result): return count == n * (n + 1) / 2 -@pytest.mark.parametrize( - "url, exp_res", - [ - ["http://example.com", False], - ["ftp://ftp.somedomain.org", False], - ["http://examplecom.", True], - ["http://.example.com", True], - ["http://example.com.", True], - ["about:newtab", True], - ["chrome://version/", True], - ], -) +@pytest.mark.parametrize('url, result', [ + ('http://user:password@hostname:1234/path?query#hash', 'user:password'), + ('http://:password@hostname:1234/path?query#hash', ':password'), + ('http://user:@hostname:1234/path?query#hash', 'user:'), + ('http://user@hostname:1234/path?query#hash', 'user'), + ('http://@hostname:1234/path?query#hash', ''), + ('http://hostname:1234/path?query#hash', None), + ('//[', ValueError('Invalid IPv6 URL')), + ('//⁈', ValueError("netloc '⁈' contains invalid characters under NFKC normalization")), +]) +def test_extract_auth(url, result): + if not isinstance(result, Exception): + assert extract_auth(url) == (result, 'http://hostname:1234/path?query#hash') + else: + try: + extract_auth(url) + except Exception as e: + assert repr(e) == repr(result) + else: + assert False, f'expected {repr(result)} to be raised' + + +@pytest.mark.parametrize('url, netloc', [ + ['http://example.com', 'example.com'], + ['example.com/#foo/bar', 'example.com'], + ['ftp://ftp.somedomain.org', 'ftp.somedomain.org'], + ['about:newtab', None], + ['chrome://version/', 'version'], + ['javascript:void(0.0)', None], + ['data:,text.with.dots', None], + ['http://[', None], # parsing error + ['http://⁈', None], # parsing error +]) +def test_get_netloc(url, netloc): + assert get_netloc(url) == netloc + + +@pytest.mark.parametrize('url, exp_res', [ + ['http://example.com', False], + ['example.com/#foo/bar', False], + ['ftp://ftp.somedomain.org', False], + ['http://examplecom.', True], # ends with a '.' + ['http://.example.com', True], # starts with a '.' + ['http://example.com.', True], # ends with a '.' + ['about:newtab', True], + ['chrome://version/', True], # contains no '.' + ['javascript:void(0.0)', True], + ['data:,text.with.dots', True], + ['http://[', True], # parsing error + ['http://⁈', True], # parsing error +]) def test_is_bad_url(url, exp_res): - """test func.""" import buku res = buku.is_bad_url(url) diff --git a/tests/test_bukuDb.py b/tests/test_bukuDb.py index 55a4a00c..e2a3d3f4 100644 --- a/tests/test_bukuDb.py +++ b/tests/test_bukuDb.py @@ -1635,18 +1635,24 @@ def test_load_firefox_database(bukuDb, firefox_db, add_pt): @pytest.mark.parametrize('ignore_case, fields, expected', [ (True, ['+id'], - ['http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', 'javascript:void(0)', 'javascript:void(1)']), + ['http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', + 'javascript:void(0)', 'javascript:void(1)', 'example.com/#']), (True, [], - ['http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', 'javascript:void(0)', 'javascript:void(1)']), - (True, ['-metadata', '+url', 'id'], - ['http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', 'http://slashdot.org', 'javascript:void(0)', 'javascript:void(1)']), - (False, ['-metadata', '+url', 'id'], - ['http://example.com/', 'javascript:void(0)', 'javascript:void(1)', 'http://www.zażółćgęśląjaźń.pl/', 'http://slashdot.org']), + ['http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', + 'javascript:void(0)', 'javascript:void(1)', 'example.com/#']), + (True, ['-metadata', '+netloc', '-url', 'id'], + ['http://www.zażółćgęśląjaźń.pl/', 'http://example.com/', 'example.com/#', + 'http://slashdot.org', 'javascript:void(1)', 'javascript:void(0)']), + (False, ['-metadata', '+netloc', 'url', 'id'], + ['example.com/#', 'http://example.com/', 'javascript:void(0)', + 'javascript:void(1)', 'http://www.zażółćgęśląjaźń.pl/', 'http://slashdot.org']), (True, ['+title', '-tags', 'description', 'index', 'uri'], - ['javascript:void(1)', 'javascript:void(0)', 'http://slashdot.org', 'http://example.com/', 'http://www.zażółćgęśląjaźń.pl/']), + ['javascript:void(1)', 'javascript:void(0)', 'http://slashdot.org', + 'http://example.com/', 'example.com/#', 'http://www.zażółćgęśląjaźń.pl/']), ]) def test_sort(bukuDb, fields, ignore_case, expected): - _bookmarks = TEST_BOOKMARKS + [(f'javascript:void({i})', 'foo', parse_tags([f'tag{i}']), 'stuff') for i in range(2)] + _bookmarks = (TEST_BOOKMARKS + [(f'javascript:void({i})', 'foo', parse_tags([f'tag{i}']), 'stuff') for i in range(2)] + + [('example.com/#', 'test', parse_tags(['test,tes,est,es']), 'a case for replace_tag test')]) bookmarks = [(i,) + tuple(x) for i, x in enumerate(_bookmarks, start=1)] shuffle(bookmarks) # making sure sorting by index works as well assert [x.url for x in bukuDb()._sort(bookmarks, fields, ignore_case=ignore_case)] == expected @@ -1654,8 +1660,8 @@ def test_sort(bukuDb, fields, ignore_case, expected): @pytest.mark.parametrize('ignore_case, fields, expected', [ (True, ['+id'], 'id ASC'), (True, [], 'id ASC'), - (False, ['-metadata', '+url', 'id'], 'metadata DESC, url ASC, id ASC'), - (True, ['-metadata', '+url', 'id'], 'LOWER(metadata) DESC, LOWER(url) ASC, id ASC'), + (False, ['-metadata', '+netloc', 'url', 'id'], 'metadata DESC, LOWER(NETLOC(url)) ASC, url ASC, id ASC'), + (True, ['-metadata', '+netloc', '-url', 'id'], 'LOWER(metadata) DESC, LOWER(NETLOC(url)) ASC, LOWER(url) DESC, id ASC'), (False, ['+title', '-tags', 'description', 'index', 'uri'], 'metadata ASC, tags DESC, desc ASC, id ASC, url ASC'), (True, ['+title', '-tags', 'description', 'index', 'uri'], 'LOWER(metadata) ASC, LOWER(tags) DESC, LOWER(desc) ASC, id ASC, LOWER(url) ASC'), @@ -1663,6 +1669,23 @@ def test_sort(bukuDb, fields, ignore_case, expected): def test_order(bukuDb, fields, ignore_case, expected): assert bukuDb()._order(fields, ignore_case=ignore_case) == expected +@pytest.mark.parametrize('order, expected', [ + (['netloc'], ['http://example.com/', 'https://example.com', '//example.com#', + 'example.com?', 'http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/']), + (['-netloc'], ['http://www.zażółćgęśląjaźń.pl/', 'http://slashdot.org', 'http://example.com/', + 'https://example.com', '//example.com#', 'example.com?']), + (['netloc', 'url'], ['//example.com#', 'example.com?', 'http://example.com/', + 'https://example.com', 'http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/']), + (['netloc', '-url'], ['https://example.com', 'http://example.com/', 'example.com?', + '//example.com#', 'http://slashdot.org', 'http://www.zażółćgęśląjaźń.pl/']), +]) +def test_order_by_netloc(bukuDb, order, expected): + bdb = bukuDb() + _EXTRA = ['https://example.com', '//example.com#', 'example.com?'] + for bookmark in (TEST_BOOKMARKS + [(url, 'test', parse_tags(['test,tes,est,es']), 'a case for replace_tag test') for url in _EXTRA]): + _add_rec(bdb, *bookmark) + assert [x.url for x in bdb.get_rec_all(order=order)] == expected + @pytest.mark.parametrize('keyword, params, expected', [ ('', {}, []), ('', {'markers': True}, []), diff --git a/tests/test_cli.py b/tests/test_cli.py index d46cc22e..d30fac47 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -143,7 +143,7 @@ def _test_add(bdb, prompt, *, add_tags=[], tag=[], tags_fetch=True, tags_in=None @pytest.mark.parametrize('np', [{}, {'np': []}]) @pytest.mark.parametrize('count', [{}, {'count': ['10']}]) @pytest.mark.parametrize('order, indices, command', [ - (['tags', '-url'], None, {'order': ['tags,-url'], 'print': []}), + (['tags', '-netloc', '+url'], None, {'order': ['tags,-netloc,+url'], 'print': []}), (['-description', '+uri'], [5, 8, 9, 10, 11, 12, 40, 41, 42], {'order': [',-description', '+uri'], 'print': ['5', '8-12', '-3']}), ])