From 1de80d932b9a779f93bc1e7b80adeab600290604 Mon Sep 17 00:00:00 2001 From: Jaspar Stach Date: Wed, 25 Oct 2023 13:38:09 +0200 Subject: [PATCH] Add: Improve UpdateHeader to remove old header lines, that are not required anymore --- pontos/updateheader/updateheader.py | 86 +++++++++++++++++++++++++++-- tests/updateheader/__init__.py | 2 +- tests/updateheader/test_header.py | 46 ++++++++++++++- 3 files changed, 126 insertions(+), 8 deletions(-) diff --git a/pontos/updateheader/updateheader.py b/pontos/updateheader/updateheader.py index c0b1a753..b17e46cb 100644 --- a/pontos/updateheader/updateheader.py +++ b/pontos/updateheader/updateheader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2022 Greenbone AG +# SPDX-FileCopyrightText: 2019-2023 Greenbone AG # # SPDX-License-Identifier: GPL-3.0-or-later # @@ -26,7 +26,7 @@ from datetime import datetime from pathlib import Path from subprocess import CalledProcessError, run -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from pontos.terminal import Terminal from pontos.terminal.null import NullTerminal @@ -53,6 +53,29 @@ "GPL-2.0-or-later", "GPL-3.0-or-later", ] +OLD_LINES = [ + "# \-\*\- coding: utf\-8 \-\*\-", + "This program is free software: you can redistribute it and/or modify", + "it under the terms of the GNU Affero General Public License as", + "published by the Free Software Foundation, either version 3 of the", + "License, or \(at your option\) any later version.", + "This program is free software; you can redistribute it and/or", + "modify it under the terms of the GNU General Public License", + "version 2 as published by the Free Software Foundation.", + "This program is free software: you can redistribute it and/or modify", + "it under the terms of the GNU General Public License as published by", + "the Free Software Foundation, either version 3 of the License, or", + "\(at your option\) any later version.", + "This program is distributed in the hope that it will be useful,", + "but WITHOUT ANY WARRANTY; without even the implied warranty of", + "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the", + "GNU Affero General Public License for more details.", + "GNU General Public License for more details.", + "You should have received a copy of the GNU Affero General Public License", + "along with this program. If not, see .", + "along with this program; if not, write to the Free Software", + "Foundation, Inc\., 51 Franklin St, Fifth Floor, Boston, MA 02110\-1301 USA\.", # noqa: E501 +] def _get_modified_year(f: Path) -> str: @@ -112,11 +135,32 @@ def _add_header( raise ValueError +def _remove_outdated( + content: str, cleanup_regexes: List[re.Pattern] +) -> Optional[str]: + """Remove lines that contain outdated copyright header ...""" + changed = False + splitted_lines = content.splitlines() + i = 0 + for line in splitted_lines[:20]: + for regex in cleanup_regexes: + if regex.match(line): + changed = True + splitted_lines.pop(i) + i = i - 1 + break + i = i + 1 + if changed: + return "\n".join(splitted_lines) + return None + + def _update_file( file: Path, regex: re.Pattern, parsed_args: Namespace, term: Terminal, + cleanup_regexes: Optional[List[re.Pattern]] = None, ) -> int: """Function to update the given file. Checks if header exists. If not it adds an @@ -173,6 +217,15 @@ def _update_file( "is not existing." ) return 1 + # old header existing - cleanup? + if cleanup_regexes: + old_content = file.read_text(encoding="utf-8") + new_content = _remove_outdated( + content=old_content, cleanup_regexes=cleanup_regexes + ) + if new_content: + file.write_text(new_content, encoding="utf-8") + print(f"{file}: Cleaned up!") # replace found header and write it to file if copyright_match and ( not copyright_match["modification_year"] @@ -201,10 +254,9 @@ def _update_file( f"{parsed_args.year}" ) - return 0 else: print(f"{file}: License Header is ok.") - return 0 + return 0 except FileNotFoundError as e: print(f"{file}: File is not existing.") raise e @@ -335,9 +387,24 @@ def _parse_args(args=None): type=FileType("r"), ) + parser.add_argument( + "--cleanup", + action="store_true", + default=False, + help="Do a cleanup: Remove lines from outdated header format", + ) + return parser.parse_args(args) +def _compile_outdated_regex() -> List[re.Pattern]: + """prepare regex patterns to remove old copyright lines""" + regexes: List[re.Pattern] = [] + for line in OLD_LINES: + regexes.append(re.compile(rf"^(([#*]|//) ?)?{line}")) + return regexes + + def main() -> None: parsed_args = _parse_args() exclude_list = [] @@ -376,10 +443,13 @@ def main() -> None: term.error("Specify files to update!") sys.exit(1) - regex = re.compile( + regex: re.Pattern = re.compile( "(SPDX-FileCopyrightText:|[Cc]opyright).*?(19[0-9]{2}|20[0-9]{2}) " f"?-? ?(19[0-9]{{2}}|20[0-9]{{2}})? ({parsed_args.company})" ) + cleanup_regexes: Optional[List[re.Pattern]] = None + if parsed_args.cleanup: + cleanup_regexes = _compile_outdated_regex() for file in files: try: @@ -387,7 +457,11 @@ def main() -> None: term.warning(f"{file}: Ignoring file from exclusion list.") else: _update_file( - file=file, regex=regex, parsed_args=parsed_args, term=term + file=file, + regex=regex, + parsed_args=parsed_args, + term=term, + cleanup_regexes=cleanup_regexes, ) except (FileNotFoundError, UnicodeDecodeError, ValueError): continue diff --git a/tests/updateheader/__init__.py b/tests/updateheader/__init__.py index ca35876b..b0b7daa8 100644 --- a/tests/updateheader/__init__.py +++ b/tests/updateheader/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 Greenbone AG +# SPDX-FileCopyrightText: 2020-2023 Greenbone AG # # SPDX-License-Identifier: GPL-3.0-or-later # diff --git a/tests/updateheader/test_header.py b/tests/updateheader/test_header.py index e9d550c6..fa6d26cb 100644 --- a/tests/updateheader/test_header.py +++ b/tests/updateheader/test_header.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 Greenbone AG +# SPDX-FileCopyrightText: 2020-2023 Greenbone AG # # SPDX-License-Identifier: GPL-3.0-or-later # @@ -28,6 +28,9 @@ from pontos.terminal.terminal import ConsoleTerminal from pontos.updateheader.updateheader import _add_header as add_header +from pontos.updateheader.updateheader import ( + _compile_outdated_regex as compile_outdated_regex, +) from pontos.updateheader.updateheader import _find_copyright as find_copyright from pontos.updateheader.updateheader import ( _get_exclude_list as get_exclude_list, @@ -36,6 +39,7 @@ _get_modified_year as get_modified_year, ) from pontos.updateheader.updateheader import _parse_args as parse_args +from pontos.updateheader.updateheader import _remove_outdated as remove_outdated from pontos.updateheader.updateheader import _update_file as update_file from pontos.updateheader.updateheader import main @@ -467,6 +471,7 @@ def test_main(self, argparser_mock): self.args.verbose = 0 self.args.log_file = None self.args.quiet = False + self.args.cleanup = False argparser_mock.return_value = self.args @@ -487,6 +492,7 @@ def test_main_never_happen(self, argparser_mock, mock_stdout): self.args.verbose = 0 self.args.log_file = None self.args.quiet = False + self.args.cleanup = False argparser_mock.return_value = self.args @@ -499,3 +505,41 @@ def test_main_never_happen(self, argparser_mock, mock_stdout): "Specify files to update!", ret, ) + + def test_remove_outdated(self): + test_content = """* This program is free software: you can redistribute it and/or modify +*it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +//License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +# modify it under the terms of the GNU General Public License +# This program is free software; you can redistribute it and/or +# version 2 as published by the Free Software Foundation. +This program is free software: you can redistribute it and/or modify""" # noqa: E501 + + compiled_regexes = compile_outdated_regex() + + new_content = remove_outdated( + content=test_content, cleanup_regexes=compiled_regexes + ) + self.assertEqual(new_content, "") + + def test_remove_outdated2(self): + test_content = """the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. +* GNU General Public License for more details. +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +# -*- coding: utf-8 -*- +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.""" # noqa: E501 + + compiled_regexes = compile_outdated_regex() + + new_content = remove_outdated( + content=test_content, cleanup_regexes=compiled_regexes + ) + self.assertEqual(new_content, "")