From e6d52d82552618dffa430adb37fa36cd66d33897 Mon Sep 17 00:00:00 2001 From: Eric Mesa Date: Mon, 20 Feb 2023 15:06:38 -0500 Subject: [PATCH] Implementing weekly culling for remote location --- .pre-commit-config.yaml | 4 ++-- snapintime/__init__.py | 2 +- snapintime/culling.py | 39 +++++++++++++++++++++--------------- snapintime/remote_culling.py | 4 ++-- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bae46d..8028ff2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort name: isort (python) diff --git a/snapintime/__init__.py b/snapintime/__init__.py index 889f167..75220c0 100644 --- a/snapintime/__init__.py +++ b/snapintime/__init__.py @@ -1,5 +1,5 @@ __author__ = "Eric Mesa" -__version__ = "1.3.1" +__version__ = "2.0.0" __license__ = "GNU GPL v3.0" __copyright__: str = "(c) 2014 - 2022 Eric Mesa" __email__: str = "ericsbinaryworld at gmail dot com" diff --git a/snapintime/culling.py b/snapintime/culling.py index d368688..bec9832 100644 --- a/snapintime/culling.py +++ b/snapintime/culling.py @@ -43,7 +43,7 @@ def get_subvols_by_date(directory: str, reg_ex, remote: bool = False, remote_loc return [subvol for subvol in subvols if reg_ex.search(subvol) is not None] -def btrfs_del(directory: str, subvols: list, remote: bool = False, remote_location: str = "") -> list: +def btrfs_del(directory: str, subvols: list, remote: bool = False, remote_location: str = "") -> list[str]: """Delete subvolumes in a given directory. :param remote_location: This should be a string like user@computer or user@IPaddress @@ -102,19 +102,19 @@ def generate_daily_cull_list(dir_to_cull: list) -> list: return list(itertools.chain.from_iterable(fourths_list)) -def cull_three_days_ago(config: dict) -> list: +def cull_three_days_ago(configuration: dict) -> list: """Cull the btrfs snapshots from 3 days ago. Take snapshots that were taken on the third day in the past and cull to 4 (max) snapshots. - :param config: The configuration file. + :param configuration: The configuration file. :returns: A list containing the results of running the commands. """ location: str = "backuplocation" three_days_ago: str = snapintime.utils.date.prior_date(datetime.now(), 3).strftime("%Y-%m-%d") three_days_ago_reg_ex = re.compile(three_days_ago) return_list = [] - for subvol in config.values(): + for subvol in configuration.values(): subvols_three_days_ago = get_subvols_by_date(subvol.get(location), three_days_ago_reg_ex) three_days_ago_culled = generate_daily_cull_list(subvols_three_days_ago) return_list.append(btrfs_del(subvol.get(location), three_days_ago_culled)) @@ -142,19 +142,26 @@ def generate_weekly_cull_list(dir_to_cull: list) -> list: return sorted_dir_to_cull -def cull_seven_days_ago(config: dict) -> list: +def cull_seven_days_ago(configuration: dict, remote: bool = False) -> list: """Cull the btrfs snapshots from 7 days ago. - :param config: The configuration file. + :param remote: True if culling on the remote server + :param configuration: The configuration file. :returns: A list containing the results of running the commands. """ seven_days_ago: str = snapintime.utils.date.prior_date(datetime.now(), 7).strftime("%Y-%m-%d") seven_days_ago_reg_ex = re.compile(seven_days_ago) return_list = [] - for subvol in config.values(): - subvols_seven_days_ago = get_subvols_by_date(subvol.get("backuplocation"), seven_days_ago_reg_ex) - seven_days_ago_culled = generate_weekly_cull_list(subvols_seven_days_ago) - return_list.append(btrfs_del(subvol.get("backuplocation"), seven_days_ago_culled)) + for subvol in configuration.values(): + location: str = "remote_subvol_dir" if remote else "backuplocation" + subvols_seven_days_ago = get_subvols_by_date(subvol.get(location), seven_days_ago_reg_ex) + if remote: + subvols_seven_days_ago = remove_protected(subvol, subvols_seven_days_ago) + if len(subvols_seven_days_ago) != 0: + seven_days_ago_culled = generate_weekly_cull_list(subvols_seven_days_ago) + return_list.append(btrfs_del(subvol.get(location), seven_days_ago_culled, remote, + remote_location=subvol.get('remote_location'))) + return return_list @@ -188,19 +195,19 @@ def remove_protected(subvol: dict, subvol_list_to_pare: list): return [subvol for subvol in subvol_list_to_pare if subvol not in protected_snapshots] -def cull_last_quarter(config: dict, remote: bool = False) -> list: +def cull_last_quarter(configuration: dict, remote: bool = False) -> list: """Cull the btrfs snapshots from quarter. Should leave 1 snapshot per week for 13 weeks. :param remote: Are we doing this on the remote system? - :param config: The configuration file. + :param configuration: The configuration file. :returns: A list containing the results of running the commands. """ last_quarter: list = snapintime.utils.date.quarterly_weeks(datetime.now()) location: str = "remote_subvol_dir" if remote else "backuplocation" return_list = [] - for subvol in config.values(): + for subvol in configuration.values(): for week in last_quarter: reg_ex_string = "" for day in week: @@ -218,18 +225,18 @@ def cull_last_quarter(config: dict, remote: bool = False) -> list: return return_list -def cull_last_year(config: dict, remote: bool = False) -> list: +def cull_last_year(configuration: dict, remote: bool = False) -> list: """Cull the btrfs snapshots from quarter. Should leave 1 snapshot per quarter for 4 quarters. :param remote: If true, we're doing this on the remote system. - :param config: The configuration file. + :param configuration: The configuration file. :returns: A list containing the results of running the commands. """ last_year: list = snapintime.utils.date.yearly_quarters(datetime.now()) return_list = [] - for subvol in config.values(): + for subvol in configuration.values(): for quarter in last_year: reg_ex_string = "" for day in quarter: diff --git a/snapintime/remote_culling.py b/snapintime/remote_culling.py index 84c3daf..867ef8b 100644 --- a/snapintime/remote_culling.py +++ b/snapintime/remote_culling.py @@ -6,10 +6,10 @@ def main(): our_config = config.import_config() # three day cull doesn't make sense because it's only 1 per day already (at least most days) - # weekly doesn't make sense either since that one is leaving 1 per day a week out - # quarterly is the first one that makes sense to implement. + weekly_cull_result = culling.cull_seven_days_ago(our_config, True) quarterly_cull_result = culling.cull_last_quarter(our_config, True) yearly_cull_result = culling.cull_last_year(our_config, True) + log.info(weekly_cull_result) log.info(quarterly_cull_result) log.info(yearly_cull_result)