From f6b4738e67c89a8174e08f298999d1fe848d995f Mon Sep 17 00:00:00 2001 From: F M Date: Sat, 4 Mar 2023 18:31:21 +0100 Subject: [PATCH 1/7] feat: select items that match a regular expression --- README.md | 8 +++++++- pwsync/common.py | 5 +++-- pwsync/console.py | 28 +++++++++++++++++++++++++--- pwsync/main.py | 15 +++++++++++++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6228712..ea240f5 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ pwsync --from demo/from.kdbx --to bitwarden # see below for an example of typical output. pwsync --from demo/from.kdbx --to demo/to.kdbx +# Do a dry-run (-d) for syncing all (-a) items that match the selection (-s) +# Item's are matched based on the concatenation of their '/' +pwsync python -mpwsync.main -d -f demo/from.kdbx -t demo/to.kdbx -a \ + -s ".*/folder 1.1/.*" -s "folder 2/.*" \ + --from-master-password=pw --to-master-password=pw + # a description of all options pwsync -h ``` @@ -198,4 +204,4 @@ An curated dump of the console output is shown below: `pwsync` is necessarily GPL3 since it (currently) depends upon the GPL3 python module `pykeepass`. -Copyright 2022 Francis Meyvis (pwsync@mikmak.fun) +Copyright 2022, 2023 Francis Meyvis (pwsync@mikmak.fun) diff --git a/pwsync/common.py b/pwsync/common.py index a618417..441c297 100644 --- a/pwsync/common.py +++ b/pwsync/common.py @@ -1,9 +1,9 @@ -# Copyright 2022 Francis Meyvis (pwsync@mikmak.fun) +# Copyright 2022, 2023 Francis Meyvis (pwsync@mikmak.fun) """common constants and helper functions""" from dataclasses import dataclass -from typing import List +from typing import List, Optional LOGGER_NAME = "PWSYNC" @@ -57,6 +57,7 @@ class PwsQueryInfo: ids: List[str] id_sep: str = ":" sync: bool = True + filters: Optional[List[str]] = None @dataclass diff --git a/pwsync/console.py b/pwsync/console.py index fb4cbce..c810f05 100644 --- a/pwsync/console.py +++ b/pwsync/console.py @@ -1,9 +1,10 @@ -# Copyright 2022 Francis Meyvis (pwsync@mikmak.fun) +# Copyright 2022, 2023 Francis Meyvis (pwsync@mikmak.fun) """interactive synchronize using the console""" import sys from difflib import SequenceMatcher +from os.path import join from typing import List, Optional from prompt_toolkit import HTML @@ -184,6 +185,22 @@ def _sync_element(kind: str, element: PwsDiffElement, to_db: PwsDatabaseClient): raise PwsUnsupported(f"_sync_element({kind})") +def _match_selector(selector, item: Optional[PwsItem]) -> bool: + if item is None: + return False + path = join(item.folder if item.folder is not None else "", item.title) + return selector.fullmatch(path) is not None + + +def _match_selectors(selectors, element): + for selector in selectors: + if _match_selector(selector, element.from_item): + return True + if _match_selector(selector, element.to_item): + return True + return False # non-matching element skipped + + def _sync_section( kind: str, query_info: PwsQueryInfo, syncer: PwsSyncer, run_options: RunOptions, to_dataset: PasswordDataset ): @@ -215,11 +232,16 @@ def _get_key_using_to_item(diff_element: PwsDiffElement): elif kind == "unchanged": return # TODO handle skipped - print_ft(HTML(_markup(f"To {kind}: {len(data)}", "info")), style=STYLE) + if query_info.filters: + filtered_data = list(filter(lambda e: _match_selectors(query_info.filters, e), data)) + else: + filtered_data = data + + print_ft(HTML(_markup(f"To {kind}: {len(filtered_data)}", "info")), style=STYLE) section_folder = "" count = 0 - for element in sorted(data, key=key_getter): + for element in sorted(filtered_data, key=key_getter): count += 1 section_folder = _print_ft_element( count, query_info, section_folder, getattr(element, props_name), element.from_item, element.to_item diff --git a/pwsync/main.py b/pwsync/main.py index b9ba5b6..40c047d 100644 --- a/pwsync/main.py +++ b/pwsync/main.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -# Copyright 2022 Francis Meyvis (pwsync@mikmak.fun) +# Copyright 2022, 2023 Francis Meyvis (pwsync@mikmak.fun) """ password sync's entry point""" +import re from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from dataclasses import dataclass from getpass import getpass @@ -125,6 +126,15 @@ def _parse_command_line(): + "or stored in the shell history buffer, etc. If left empty, it is prompted for on the command line", ) + parser.add_argument( + "-s", + "--select", + action="append", + nargs="*", + default=[], + help="Select the items to sync by matching their folder/title value with the regular expression.", + ) + parser.add_argument("-U", "--auto-update", action="store_true", help="automatically update all entries") parser.add_argument("-C", "--auto-create", action="store_true", help="automatically create all entries") @@ -211,7 +221,8 @@ def main(): logger.addHandler(file_handler) logger.propagate = False # do not further propagate to the root handler (stdout) - query_info = PwsQueryInfo(args.id.split(","), args.id_sep, args.sync) + filters = list(map(re.compile, map(lambda i: i[0], args.select))) + query_info = PwsQueryInfo(args.id.split(","), args.id_sep, args.sync, filters) access = _AccessInfo(args.from_username, args.from_secret, args.from_master_password) from_dataset = _open_password_db(query_info, "from", getattr(args, "from"), access) From 5471c3beda4d4c5cf2851140a4554d61810912d3 Mon Sep 17 00:00:00 2001 From: F M Date: Sat, 4 Mar 2023 18:42:27 +0100 Subject: [PATCH 2/7] fix: bump python version in CI --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff3608b..947087f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8"] + python-version: ["3.9"] fail-fast: false steps: - uses: actions/checkout@v2 From c60944acf9362f71edd4bf379bdc97582c6ce476 Mon Sep 17 00:00:00 2001 From: F M Date: Sat, 4 Mar 2023 18:56:31 +0100 Subject: [PATCH 3/7] fix: python version --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 947087f..f29869f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt-get install -y --upgrade git build-essential python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-dev curl + sudo apt-get install -y --upgrade git build-essential python3-venv python3-dev curl curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm From 38765557797dfdfd4d45e0cb95b0fe6001864ac5 Mon Sep 17 00:00:00 2001 From: F M Date: Sun, 5 Mar 2023 07:44:19 +0100 Subject: [PATCH 4/7] fix: manual typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea240f5..a79df82 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ pwsync --from demo/from.kdbx --to demo/to.kdbx # Do a dry-run (-d) for syncing all (-a) items that match the selection (-s) # Item's are matched based on the concatenation of their '/' -pwsync python -mpwsync.main -d -f demo/from.kdbx -t demo/to.kdbx -a \ +python -mpwsync.main -d -f demo/from.kdbx -t demo/to.kdbx -a \ -s ".*/folder 1.1/.*" -s "folder 2/.*" \ --from-master-password=pw --to-master-password=pw From ac1389bf18efbe2a4327e15f3ec4785140d2e93c Mon Sep 17 00:00:00 2001 From: F M Date: Sun, 5 Mar 2023 07:45:27 +0100 Subject: [PATCH 5/7] chore: update dependency versions --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c4e3036..44d0808 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ url="https://github.com/aptly-io/pwsync", classifiers=[ "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: POSIX :: Linux", # TODO test on other OSes @@ -34,8 +33,8 @@ packages=find_packages(), install_requires=[ "pykeepass==4.0.3", - "diffsync==1.6.0", - "prompt-toolkit==3.0.30", + "diffsync==1.7.0", + "prompt-toolkit==3.0.38", ], extras_require={ "dev": [ From 79ea419afb315a4dbd421ad522bf683191b43896 Mon Sep 17 00:00:00 2001 From: F M Date: Sun, 5 Mar 2023 07:46:54 +0100 Subject: [PATCH 6/7] chore: reduce lint errors --- pwsync/__init__.py | 4 ++-- pwsync/bw_cli_wrapper.py | 2 +- pwsync/console.py | 26 +++++++++++++++++--------- pwsync/sync.py | 2 +- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pwsync/__init__.py b/pwsync/__init__.py index f65d1be..07a6e2b 100644 --- a/pwsync/__init__.py +++ b/pwsync/__init__.py @@ -4,11 +4,11 @@ import sys -assert sys.version_info >= (3, 7) - from .bw_cli_wrapper import BitwardenClientWrapper from .common import PwsDuplicate, PwsMissingOrganization, PwsUnsupported from .dataset import PasswordDataset from .item import PwsItem from .kp_db_cli import KeepassDatabaseClient from .sync import PwsSyncer + +assert sys.version_info >= (3, 7) diff --git a/pwsync/bw_cli_wrapper.py b/pwsync/bw_cli_wrapper.py index 7781b03..74e2319 100644 --- a/pwsync/bw_cli_wrapper.py +++ b/pwsync/bw_cli_wrapper.py @@ -190,7 +190,7 @@ def _check_output( input_value=None, ): try: - result_json = check_output(cmd, input=input_value, env=self._env) + result_json = check_output(cmd, input=input_value, env=self._env).decode("utf-8") self._logger.debug("cmd: %s, result: %s", cmd, result_json) return json.loads(result_json) except CalledProcessError as exc: diff --git a/pwsync/console.py b/pwsync/console.py index c810f05..b047d13 100644 --- a/pwsync/console.py +++ b/pwsync/console.py @@ -201,9 +201,8 @@ def _match_selectors(selectors, element): return False # non-matching element skipped -def _sync_section( - kind: str, query_info: PwsQueryInfo, syncer: PwsSyncer, run_options: RunOptions, to_dataset: PasswordDataset -): +def _switch_kind(kind: str, query_info: PwsQueryInfo, syncer): + def _get_key_using_from_item(diff_element: PwsDiffElement): return diff_element.from_item.make_id(query_info) if diff_element.from_item else "" @@ -228,20 +227,29 @@ def _get_key_using_to_item(diff_element: PwsDiffElement): key_getter = _get_key_using_from_item props_name = "add_props" elif kind == "skipped": - return # TODO handle skipped + raise NotImplementedError("kind 'skipped' not handled") elif kind == "unchanged": - return # TODO handle skipped + raise NotImplementedError("kind 'unchanged' not handled") if query_info.filters: filtered_data = list(filter(lambda e: _match_selectors(query_info.filters, e), data)) else: filtered_data = data - print_ft(HTML(_markup(f"To {kind}: {len(filtered_data)}", "info")), style=STYLE) + return filtered_data, key_getter, props_name + + +def _sync_section( + kind: str, query_info: PwsQueryInfo, syncer: PwsSyncer, run_options: RunOptions, to_dataset: PasswordDataset +): + data, key_getter, props_name = _switch_kind(kind, query_info, syncer) + + print_ft(HTML(_markup(f"To {kind}: {len(data)}", "info")), style=STYLE) section_folder = "" count = 0 - for element in sorted(filtered_data, key=key_getter): + + for element in sorted(data, key=key_getter): count += 1 section_folder = _print_ft_element( count, query_info, section_folder, getattr(element, props_name), element.from_item, element.to_item @@ -271,5 +279,5 @@ def console_sync(query_info: PwsQueryInfo, syncer: PwsSyncer, run_options: RunOp _sync_section("update", query_info, syncer, run_options, to_dataset) _sync_section("create", query_info, syncer, run_options, to_dataset) _sync_section("delete", query_info, syncer, run_options, to_dataset) - _sync_section("skipped", query_info, syncer, run_options, to_dataset) - _sync_section("unchanged", query_info, syncer, run_options, to_dataset) + # _sync_section("skipped", query_info, syncer, run_options, to_dataset) # TODO handle skipped + # _sync_section("unchanged", query_info, syncer, run_options, to_dataset) # TODO handle unchanged diff --git a/pwsync/sync.py b/pwsync/sync.py index d26432d..d38c645 100644 --- a/pwsync/sync.py +++ b/pwsync/sync.py @@ -90,7 +90,7 @@ def sync(self): elif diff_element.action == DiffSyncActions.UPDATE: self._update(diff_element) else: - raise Exception(f"Unexpected action: {diff_element.action}") + raise NotImplementedError(f"Unexpected action: {diff_element.action}") self._logger.info("diff.summary: %s", diff.summary()) self._logger.debug("self.creates: %s", self.creates) From 8c511bf1d71d407c554e1e29697ee9104466a2f6 Mon Sep 17 00:00:00 2001 From: F M Date: Sun, 5 Mar 2023 08:45:05 +0100 Subject: [PATCH 7/7] chore: reformat --- pwsync/console.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pwsync/console.py b/pwsync/console.py index b047d13..19c09a2 100644 --- a/pwsync/console.py +++ b/pwsync/console.py @@ -202,7 +202,6 @@ def _match_selectors(selectors, element): def _switch_kind(kind: str, query_info: PwsQueryInfo, syncer): - def _get_key_using_from_item(diff_element: PwsDiffElement): return diff_element.from_item.make_id(query_info) if diff_element.from_item else ""