From 7472c51e832f68c1454036be0cb10bd10130b864 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 20 Nov 2024 18:35:37 +1300 Subject: [PATCH 01/50] feat: support deb822 style source specification Ubuntu 24.04 adopts the deb822 style source specification. Such files are listed in /etc/apt/sources.list.d/*sources, and allow the specification of sources in a multi-line format --- lib/charms/operator_libs_linux/v0/apt.py | 169 ++++++++++++++++++++++- 1 file changed, 163 insertions(+), 6 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index b8913c0e..db8e52cd 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -102,6 +102,7 @@ import fileinput import glob +import itertools import logging import os import re @@ -109,7 +110,7 @@ from collections.abc import Mapping from enum import Enum from subprocess import PIPE, CalledProcessError, check_output -from typing import Iterable, List, Optional, Tuple, Union +from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -122,7 +123,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 14 +LIBPATCH = 15 VALID_SOURCE_TYPES = ("deb", "deb-src") @@ -1198,7 +1199,7 @@ class RepositoryMapping(Mapping): """ def __init__(self): - self._repository_map = {} + self._repository_map: Dict[str, DebianRepository] = {} # Repositories that we're adding -- used to implement mode param self.default_file = "/etc/apt/sources.list" @@ -1210,6 +1211,9 @@ def __init__(self): for file in glob.iglob("/etc/apt/sources.list.d/*.list"): self.load(file) + for file in glob.iglob("/etc/apt/sources.list.d/*.sources"): + self.load_deb822(file) + def __contains__(self, key: str) -> bool: """Magic method for checking presence of repo in mapping.""" return key in self._repository_map @@ -1231,13 +1235,13 @@ def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None self._repository_map[repository_uri] = repository def load(self, filename: str): - """Load a repository source file into the cache. + """Load a one-line-style format repository source file into the cache. Args: filename: the path to the repository file """ - parsed = [] - skipped = [] + parsed: List[int] = [] + skipped: List[int] = [] with open(filename, "r") as f: for n, line in enumerate(f): try: @@ -1314,6 +1318,159 @@ def _parse(line: str, filename: str) -> DebianRepository: else: raise InvalidSourceError("An invalid sources line was found in %s!", filename) + def load_deb822(self, filename: str) -> None: + """Load a deb822 format repository source file into the cache. + + Args: + filename: the path to the repository file + + In contrast to one-line-style, the deb822 format specifies a repository + using a multi-line paragraph. Paragraphs are separated by whitespace, + and each definition consists of lines that are either key: value pairs, + or continuations of the previous value. + + Read more about the deb822 format here: + https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html + For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: + /etc/apt/sources.list.d/ubuntu.sources + + The semantics of `load_deb822` slightly different to `load`: + `load` calls `_parse`, with reads a commented out line as an entry that is not enabled + `load_deb822` strips out comments entirely when parsing a file into paragraphs, and + `_parse_deb822_paragraph` assumes that comments have been removed entirely, + instead reading the 'Enabled' key to determine if an entry is enabled + """ + parsed: List[int] = [] + skipped: List[InvalidSourceError] = [] + with open(filename, "r") as f: + for line_number, paragraph in self._iter_paragraphs(f): + try: + repos = self._parse_deb822_paragraph( + paragraph, filename=filename, line_number=line_number + ) + except InvalidSourceError as e: + skipped.append(e) + continue + for repo in repos: + repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) + self._repository_map[repo_identifier] = repo + parsed.append(line_number) + + if skipped: + logger.debug( + "the following errors were encountered when reading deb822 format sources:\n%s", + "\n".join(str(error) for error in skipped), + ) + + if parsed: + logger.info("parsed %d apt package repositories", len(parsed)) + else: + raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) + + @staticmethod + def _iter_paragraphs(lines: Iterable[str]) -> Iterator[Tuple[int, List[str]]]: + current: Optional[Tuple[int, List[str]]] = None + for n, line in enumerate(lines): # 0 indexed line numbers, following `load` + if line.startswith("#"): + continue + if not line: + if current is not None: + yield current + current = None + continue + if current is None: + current = (n, []) + _line_number, paragraph_lines = current + paragraph_lines.append(line) + if current is not None: + yield current + + @staticmethod + def _parse_deb822_paragraph( + lines: List[str], + filename: str, + line_number: int, + ) -> List[DebianRepository]: + """Parse a list of lines forming a deb822 style repository definition. + + Args: + lines: a list of lines forming a deb822 paragraph + filename: the name of the file being read (for DebianRepository and errors) + line_number: the line number the paragraph starts on (for errors) + + Raises: + InvalidSourceError if the source type is unknown or contains malformed entries + """ + parts: Dict[str, List[str]] = {} + current = None + for line in lines: + if line.startswith(" "): # continuation of previous key's value + assert current is not None + parts[current].append(line) + continue + assert not line.startswith("#") # comments should be stripped out + raw_key, _, raw_value = line.partition(":") + current = raw_key.strip() + parts[current] = [raw_value.lstrip()] + options = {k: "\n".join(v) for k, v in parts.items()} + + enabled_field = options.pop("Enabled", "yes") + if enabled_field == "yes": + enabled = True + elif enabled_field == "no": + enabled = False + else: + raise InvalidSourceError( + "Malformed value for entry 'Enabled' for paragraph starting on line %s in %s!", + line_number, + filename, + ) + + gpg_key = options.pop("Signed-By", "") + repotypes = options.pop("Types").split() + uris = options.pop("URIs").split() + suites = options.pop("Suites").split() + + components: List[str] + if len(suites) == 1 and suites[0].endswith("/"): + if "Components" in options: + raise InvalidSourceError( + ( + "For paragraph starting on line %s in %s," + " since 'Suites' specifies a path relative to" + " 'URIs', 'Components' must be ommitted.", + ), + line_number, + filename, + ) + components = [] + else: + if "Components" not in options: + raise InvalidSourceError( + ( + "For paragraph starting on line %s in %s," + " since 'Suites' does not specify a path relative to" + " 'URIs', 'Components' must be present.", + ), + line_number, + filename, + ) + components = options.pop("Components").split() + + return [ + DebianRepository( + enabled=enabled, + repotype=repotype, + uri=uri, + release=suite, + groups=components, + filename=filename, + gpg_key_filename=gpg_key, # TODO: gpg_key can be a literal key, not just a filename + options=options, + ) + for repotype, uri, suite in itertools.product(repotypes, uris, suites) + ] + def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: """Add a new repository to the system. From 66f42cdbbec3c02f5e847620f925b0852d72ac7c Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 11:11:27 +1300 Subject: [PATCH 02/50] refactor: separate file reading and object updating from parsing lines --- lib/charms/operator_libs_linux/v0/apt.py | 85 ++++++++++++++---------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index db8e52cd..3e6cedf9 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1337,59 +1337,70 @@ def load_deb822(self, filename: str) -> None: The semantics of `load_deb822` slightly different to `load`: `load` calls `_parse`, with reads a commented out line as an entry that is not enabled `load_deb822` strips out comments entirely when parsing a file into paragraphs, and - `_parse_deb822_paragraph` assumes that comments have been removed entirely, + assumes that comments have been removed when parsing individual paragraphs/entry, instead reading the 'Enabled' key to determine if an entry is enabled """ - parsed: List[int] = [] - skipped: List[InvalidSourceError] = [] with open(filename, "r") as f: - for line_number, paragraph in self._iter_paragraphs(f): - try: - repos = self._parse_deb822_paragraph( - paragraph, filename=filename, line_number=line_number - ) - except InvalidSourceError as e: - skipped.append(e) - continue - for repo in repos: - repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) - self._repository_map[repo_identifier] = repo - parsed.append(line_number) + repos, errors = self._parse_deb822_lines(f, filename=filename) - if skipped: + for repo in repos: + repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) + self._repository_map[repo_identifier] = repo + + if errors: logger.debug( "the following errors were encountered when reading deb822 format sources:\n%s", - "\n".join(str(error) for error in skipped), + "\n".join(str(e) for e in errors), ) - if parsed: - logger.info("parsed %d apt package repositories", len(parsed)) + if repos: + logger.info("parsed %d apt package repositories", len(repos)) else: raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) + @classmethod + def _parse_deb822_lines( + cls, + lines: Iterable[str], + filename: str = "", + ) -> Tuple[List[DebianRepository], List[InvalidSourceError]]: + """Parse lines from a deb822 file into a list of repos and a list of errors.""" + repositories: List[DebianRepository] = [] + errors: List[InvalidSourceError] = [] + for line_number, paragraph in cls._iter_deb822_paragraphs(lines): + try: + repos = cls._parse_deb822_paragraph( + paragraph, filename=filename, line_number=line_number + ) + except InvalidSourceError as e: + errors.append(e) + else: + repositories.extend(repos) + return repositories, errors + @staticmethod - def _iter_paragraphs(lines: Iterable[str]) -> Iterator[Tuple[int, List[str]]]: - current: Optional[Tuple[int, List[str]]] = None + def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[Tuple[int, List[str]]]: + current_paragraph: Optional[Tuple[int, List[str]]] = None for n, line in enumerate(lines): # 0 indexed line numbers, following `load` if line.startswith("#"): continue - if not line: - if current is not None: - yield current - current = None + if not line: # blank lines separate paragraphs + if current_paragraph is not None: + yield current_paragraph + current_paragraph = None continue - if current is None: - current = (n, []) - _line_number, paragraph_lines = current + if current_paragraph is None: + current_paragraph = (n, []) + _line_number, paragraph_lines = current_paragraph paragraph_lines.append(line) - if current is not None: - yield current + if current_paragraph is not None: + yield current_paragraph @staticmethod def _parse_deb822_paragraph( lines: List[str], - filename: str, - line_number: int, + filename: str = "", + line_number: Optional[int] = None, ) -> List[DebianRepository]: """Parse a list of lines forming a deb822 style repository definition. @@ -1401,6 +1412,10 @@ def _parse_deb822_paragraph( Raises: InvalidSourceError if the source type is unknown or contains malformed entries """ + line_number_msg = ( + str(line_number) if line_number is not None else "(no line number specified)" + ) + parts: Dict[str, List[str]] = {} current = None for line in lines: @@ -1422,7 +1437,7 @@ def _parse_deb822_paragraph( else: raise InvalidSourceError( "Malformed value for entry 'Enabled' for paragraph starting on line %s in %s!", - line_number, + line_number_msg, filename, ) @@ -1440,7 +1455,7 @@ def _parse_deb822_paragraph( " since 'Suites' specifies a path relative to" " 'URIs', 'Components' must be ommitted.", ), - line_number, + line_number_msg, filename, ) components = [] @@ -1452,7 +1467,7 @@ def _parse_deb822_paragraph( " since 'Suites' does not specify a path relative to" " 'URIs', 'Components' must be present.", ), - line_number, + line_number_msg, filename, ) components = options.pop("Components").split() From 20b0f9ed3d6a2e60f76cf236788e57ac04b751a0 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 11:39:44 +1300 Subject: [PATCH 03/50] refactor: preserve line nos when parsing deb822 lines into paragraphs --- lib/charms/operator_libs_linux/v0/apt.py | 80 +++++++++++++----------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 3e6cedf9..6e43b93c 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1367,11 +1367,9 @@ def _parse_deb822_lines( """Parse lines from a deb822 file into a list of repos and a list of errors.""" repositories: List[DebianRepository] = [] errors: List[InvalidSourceError] = [] - for line_number, paragraph in cls._iter_deb822_paragraphs(lines): + for paragraph in cls._iter_deb822_paragraphs(lines): try: - repos = cls._parse_deb822_paragraph( - paragraph, filename=filename, line_number=line_number - ) + repos = cls._parse_deb822_paragraph(paragraph, filename=filename) except InvalidSourceError as e: errors.append(e) else: @@ -1379,46 +1377,38 @@ def _parse_deb822_lines( return repositories, errors @staticmethod - def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[Tuple[int, List[str]]]: - current_paragraph: Optional[Tuple[int, List[str]]] = None + def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: + current_paragraph: List[Tuple[int, str]] = [] for n, line in enumerate(lines): # 0 indexed line numbers, following `load` if line.startswith("#"): continue if not line: # blank lines separate paragraphs - if current_paragraph is not None: + if current_paragraph: yield current_paragraph - current_paragraph = None + current_paragraph = [] continue - if current_paragraph is None: - current_paragraph = (n, []) - _line_number, paragraph_lines = current_paragraph - paragraph_lines.append(line) - if current_paragraph is not None: + current_paragraph.append((n, line)) + if current_paragraph: yield current_paragraph @staticmethod def _parse_deb822_paragraph( - lines: List[str], + lines: List[Tuple[int, str]], filename: str = "", - line_number: Optional[int] = None, ) -> List[DebianRepository]: - """Parse a list of lines forming a deb822 style repository definition. + """Parse a list of numbered lines forming a deb822 style repository definition. Args: - lines: a list of lines forming a deb822 paragraph + lines: a list of numbered lines forming a deb822 paragraph filename: the name of the file being read (for DebianRepository and errors) - line_number: the line number the paragraph starts on (for errors) Raises: InvalidSourceError if the source type is unknown or contains malformed entries """ - line_number_msg = ( - str(line_number) if line_number is not None else "(no line number specified)" - ) - parts: Dict[str, List[str]] = {} + line_numbers: Dict[str, int] = {} current = None - for line in lines: + for n, line in lines: if line.startswith(" "): # continuation of previous key's value assert current is not None parts[current].append(line) @@ -1427,6 +1417,7 @@ def _parse_deb822_paragraph( raw_key, _, raw_value = line.partition(":") current = raw_key.strip() parts[current] = [raw_value.lstrip()] + line_numbers[current] = n options = {k: "\n".join(v) for k, v in parts.items()} enabled_field = options.pop("Enabled", "yes") @@ -1436,9 +1427,15 @@ def _parse_deb822_paragraph( enabled = False else: raise InvalidSourceError( - "Malformed value for entry 'Enabled' for paragraph starting on line %s in %s!", - line_number_msg, - filename, + ( + "Bad value '{value}' for entry 'Enabled' (line {enabled_line})" + " in file {file}. If 'Enabled' is present it must be one of" + " yes or no (if absent it defaults to yes)." + ).format( + value=enabled_field, + enabled_line=line_numbers["Enabled"], + file=filename, + ) ) gpg_key = options.pop("Signed-By", "") @@ -1451,24 +1448,31 @@ def _parse_deb822_paragraph( if "Components" in options: raise InvalidSourceError( ( - "For paragraph starting on line %s in %s," - " since 'Suites' specifies a path relative to" - " 'URIs', 'Components' must be ommitted.", - ), - line_number_msg, - filename, + "Since 'Suites' (line {suites_line}) specifies" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' (line {components_line}) must be ommitted" + " (in file {file})." + ).format( + suites_line=line_numbers["Suites"], + uris_line=line_numbers["URIs"], + components_line=line_numbers["Components"], + file=filename, + ) ) components = [] else: if "Components" not in options: raise InvalidSourceError( ( - "For paragraph starting on line %s in %s," - " since 'Suites' does not specify a path relative to" - " 'URIs', 'Components' must be present.", - ), - line_number_msg, - filename, + "Since 'Suites' (line {suites_line}) does not specify" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be present in this paragraph" + " (in file {file})." + ).format( + suites_line=line_numbers["Suites"], + uris_line=line_numbers["URIs"], + file=filename, + ) ) components = options.pop("Components").split() From 0e6c8b302dcdaeb13096ad6deffde8963792ed31 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 12:06:37 +1300 Subject: [PATCH 04/50] refactor: move turning numbered lines into options dict to helper --- lib/charms/operator_libs_linux/v0/apt.py | 38 ++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 6e43b93c..953558de 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1391,8 +1391,9 @@ def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[List[Tuple[int, st if current_paragraph: yield current_paragraph - @staticmethod + @classmethod def _parse_deb822_paragraph( + cls, lines: List[Tuple[int, str]], filename: str = "", ) -> List[DebianRepository]: @@ -1405,20 +1406,7 @@ def _parse_deb822_paragraph( Raises: InvalidSourceError if the source type is unknown or contains malformed entries """ - parts: Dict[str, List[str]] = {} - line_numbers: Dict[str, int] = {} - current = None - for n, line in lines: - if line.startswith(" "): # continuation of previous key's value - assert current is not None - parts[current].append(line) - continue - assert not line.startswith("#") # comments should be stripped out - raw_key, _, raw_value = line.partition(":") - current = raw_key.strip() - parts[current] = [raw_value.lstrip()] - line_numbers[current] = n - options = {k: "\n".join(v) for k, v in parts.items()} + options, line_numbers = cls._get_deb822_options(lines) enabled_field = options.pop("Enabled", "yes") if enabled_field == "yes": @@ -1490,6 +1478,26 @@ def _parse_deb822_paragraph( for repotype, uri, suite in itertools.product(repotypes, uris, suites) ] + @staticmethod + def _get_deb822_options( + lines: Iterable[Tuple[int, str]] + ) -> Tuple[Dict[str, str], Dict[str, int]]: + parts: Dict[str, List[str]] = {} + line_numbers: Dict[str, int] = {} + current = None + for n, line in lines: + assert "#" not in line # comments should be stripped out + if line.startswith(" "): # continuation of previous key's value + assert current is not None + parts[current].append(line.rstrip()) # preserve indent + continue + raw_key, _, raw_value = line.partition(":") + current = raw_key.strip() + parts[current] = [raw_value.strip()] + line_numbers[current] = n + options = {k: "\n".join(v) for k, v in parts.items()} + return options, line_numbers + def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: """Add a new repository to the system. From 8dbd1589a186bb5d12aba989e83d525a36100ba3 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 13:10:06 +1300 Subject: [PATCH 05/50] feat: support inline comments in deb822 source files --- lib/charms/operator_libs_linux/v0/apt.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 953558de..f33dc51d 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1378,16 +1378,21 @@ def _parse_deb822_lines( @staticmethod def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: + """Given lines from a deb822 format file, yield paragraphs. + + A paragraph is a list of numbered lines that make up a source entry, + with comments stripped out (but accounted for in line numbering). + """ current_paragraph: List[Tuple[int, str]] = [] for n, line in enumerate(lines): # 0 indexed line numbers, following `load` - if line.startswith("#"): - continue if not line: # blank lines separate paragraphs if current_paragraph: yield current_paragraph current_paragraph = [] continue - current_paragraph.append((n, line)) + content, _delim, _comment = line.partition("#") + if content.strip(): # skip (potentially indented) comment line + current_paragraph.append((n, content.rstrip())) # preserve indent if current_paragraph: yield current_paragraph From 9d9c9fa3ebd31b3a914246cf31cff01d79ae97b5 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 13:12:31 +1300 Subject: [PATCH 06/50] tests: add some unit tests for _iter_deb822_paragraphs --- tests/unit/test_apt.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index 99c43ca5..99573757 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -172,6 +172,41 @@ Description-md5: e7f99df3aa92cf870d335784e155ec33 """ +ubuntu_sources_deb822 = """ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +""" + +ubuntu_sources_deb822_with_comments = """ +Components: main restricted universe multiverse # this lib doesn't care about order +Types: deb # this could include deb-src as well or instead +URIs: http://nz.archive.ubuntu.com/ubuntu/ + # there can be multiple space separated URIs + # sources are specified in priority order + # apt does some de-duplication of sources after parsing too +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +# let's make this insecure! (jk, just testing parsing) +Suites: noble noble-updates noble-backports + +Foo: Bar # this is a separate (malformed) entry + +#Types: deb +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +## disable security updates while we're at it +""" + class TestApt(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") @@ -383,6 +418,41 @@ def test_can_parse_epoch_and_version(self): ("2", "9.8-7ubuntu6"), apt.DebianPackage._get_epoch_from_version("2:9.8-7ubuntu6") ) + def test_iter_deb822_paragraphs_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) + assert paras == [ + [ + (0, "Types: deb"), + (1, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), + (2, "Suites: noble noble-updates noble-backports"), + (3, "Components: main restricted universe multiverse"), + (4, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), + ], + [ + (6, "Types: deb"), + (7, "URIs: http://security.ubuntu.com/ubuntu"), + (8, "Suites: noble-security"), + (9, "Components: main restricted universe multiverse"), + (10, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), + ], + ] + + def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) + assert paras == [ + [ + (0, "Components: main restricted universe multiverse"), + (1, "Types: deb"), + (2, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), + (8, "Suites: noble noble-updates noble-backports"), + ], + [ + (10, "Foo: Bar"), + ], + ] + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From 5a7c22eba9370b7e6449366381a700000e6dd145 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 14:22:37 +1300 Subject: [PATCH 07/50] fix: raise an InvalidSourceError on missing required deb822 keys --- lib/charms/operator_libs_linux/v0/apt.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index f33dc51d..da1a324f 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1432,9 +1432,20 @@ def _parse_deb822_paragraph( ) gpg_key = options.pop("Signed-By", "") - repotypes = options.pop("Types").split() - uris = options.pop("URIs").split() - suites = options.pop("Suites").split() + + try: + repotypes = options.pop("Types").split() + uris = options.pop("URIs").split() + suites = options.pop("Suites").split() + except KeyError as e: + [key] = e.args + raise InvalidSourceError( + "Missing key '{key}' for entry starting on line {line} in {file}.".format( + key=key, + line=min(line_numbers.values()), + file=filename, + ) + ) components: List[str] if len(suites) == 1 and suites[0].endswith("/"): From bf26ce0b10cdc29d3e698639e953c70d726f4f79 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 14:56:54 +1300 Subject: [PATCH 08/50] tests: add some unit tests for _get_deb822_options --- tests/unit/test_apt.py | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index 99573757..18a9ffa1 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -453,6 +453,70 @@ def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): ], ] + def test_get_deb822_options_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) + opts = [ + apt.RepositoryMapping._get_deb822_options(p) + for p in paras + ] + opts_0, opts_1 = opts + opts_0_options, opts_0_line_numbers = opts_0 + opts_1_options, opts_1_line_numbers = opts_1 + assert opts_0_options == { + 'Types': 'deb', + 'URIs': 'http://nz.archive.ubuntu.com/ubuntu/', + 'Components': 'main restricted universe multiverse', + 'Suites': 'noble noble-updates noble-backports', + 'Signed-By': '/usr/share/keyrings/ubuntu-archive-keyring.gpg', + } + assert opts_0_line_numbers == { + 'Types': 0, + 'URIs': 1, + 'Suites': 2, + 'Components': 3, + 'Signed-By': 4, + } + assert opts_1_options == { + 'Types': 'deb', + 'URIs': 'http://security.ubuntu.com/ubuntu', + 'Components': 'main restricted universe multiverse', + 'Suites': 'noble-security', + 'Signed-By': '/usr/share/keyrings/ubuntu-archive-keyring.gpg', + } + assert opts_1_line_numbers == { + 'Types': 6, + 'URIs': 7, + 'Suites': 8, + 'Components': 9, + 'Signed-By': 10, + } + + def test_get_deb822_options_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) + opts = [ + apt.RepositoryMapping._get_deb822_options(p) + for p in paras + ] + opts_0, opts_1 = opts + opts_0_options, opts_0_line_numbers = opts_0 + opts_1_options, opts_1_line_numbers = opts_1 + assert opts_0_options == { + 'Components': 'main restricted universe multiverse', + 'Types': 'deb', + 'URIs': 'http://nz.archive.ubuntu.com/ubuntu/', + 'Suites': 'noble noble-updates noble-backports', + } + assert opts_0_line_numbers == { + 'Components': 0, + 'Types': 1, + 'URIs': 2, + 'Suites': 8, + } + assert opts_1_options == {'Foo': 'Bar'} + assert opts_1_line_numbers == {'Foo': 10} + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From d43074b0cbbe97d2f034bd2533c6def45526b2df Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 16:21:21 +1300 Subject: [PATCH 09/50] fix: correctly handle newline terminated lines in paragraph iteration --- lib/charms/operator_libs_linux/v0/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index da1a324f..7394cdb4 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1385,7 +1385,7 @@ def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[List[Tuple[int, st """ current_paragraph: List[Tuple[int, str]] = [] for n, line in enumerate(lines): # 0 indexed line numbers, following `load` - if not line: # blank lines separate paragraphs + if not line.strip(): # blank lines separate paragraphs if current_paragraph: yield current_paragraph current_paragraph = [] From 484b106c96c2b7ff8b6f0fd6fb1c2444a2aaf7a3 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 16:24:12 +1300 Subject: [PATCH 10/50] tests: add some unit tests for _iter_deb822_paragraphs --- tests/unit/test_apt.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index 18a9ffa1..b3564c4f 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -517,6 +517,46 @@ def test_get_deb822_options_w_comments(self): assert opts_1_options == {'Foo': 'Bar'} assert opts_1_line_numbers == {'Foo': 10} + def test_parse_deb822_paragraph_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + main, security = apt.RepositoryMapping._iter_deb822_paragraphs(lines) + repos = apt.RepositoryMapping._parse_deb822_paragraph(main) + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + repos = apt.RepositoryMapping._parse_deb822_paragraph(security) + assert len(repos) == 1 + [repo] = repos + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://security.ubuntu.com/ubuntu" + assert repo.release == "noble-security" + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + + def test_parse_deb822_paragraph_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + ok_para, bad_para = apt.RepositoryMapping._iter_deb822_paragraphs(lines) + repos = apt.RepositoryMapping._parse_deb822_paragraph(ok_para) + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "" + with self.assertRaises(apt.InvalidSourceError): + apt.RepositoryMapping._parse_deb822_paragraph(bad_para) + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From 86e78b5234ae89d013634012db430eb029e02d2f Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 16:25:03 +1300 Subject: [PATCH 11/50] tests: add some tests for _parse_deb822_lines --- tests/unit/test_apt.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index b3564c4f..b3dc966f 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -557,6 +557,21 @@ def test_parse_deb822_paragraph_w_comments(self): with self.assertRaises(apt.InvalidSourceError): apt.RepositoryMapping._parse_deb822_paragraph(bad_para) + def test_parse_deb822_lines_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + print(repos[0].__dict__) + assert len(repos) == 4 + assert not errors + + def test_parse_deb822_lines_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + assert len(repos) == 3 + assert len(errors) == 1 + [error] = errors + assert isinstance(error, apt.InvalidSourceError) + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From 2dd9dcd17f8c9da3b1c80a83763540158e3c342b Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 16:44:48 +1300 Subject: [PATCH 12/50] feat: add the number of errors to the debug output in load_deb822 --- lib/charms/operator_libs_linux/v0/apt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 7394cdb4..378ee8cb 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1349,7 +1349,8 @@ def load_deb822(self, filename: str) -> None: if errors: logger.debug( - "the following errors were encountered when reading deb822 format sources:\n%s", + "the following %d error(s) were encountered when reading deb822 format sources:\n%s", + len(errors), "\n".join(str(e) for e in errors), ) From 6c39d86017cb1896e014eaf51f625dd4740cf2f3 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 16:45:57 +1300 Subject: [PATCH 13/50] tests: add some unit tests for load_deb822 --- tests/unit/test_apt.py | 45 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index b3dc966f..834ac0b6 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -3,7 +3,7 @@ import subprocess import unittest -from unittest.mock import patch +from unittest.mock import mock_open, patch, ANY from charms.operator_libs_linux.v0 import apt @@ -572,6 +572,49 @@ def test_parse_deb822_lines_w_comments(self): [error] = errors assert isinstance(error, apt.InvalidSourceError) + def test_load_deb822_ubuntu_sources(self): + with ( + patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch('glob.iglob', new=lambda s: []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + ): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + mopen = mock_open(read_data=ubuntu_sources_deb822) + mopen.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + with patch('builtins.open', new=mopen): + repository_mapping.load_deb822("") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + "deb-http://security.ubuntu.com/ubuntu-noble-security", + ] + + def test_load_deb822_w_comments(self): + with ( + patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch('glob.iglob', new=lambda s: []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + ): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + mopen = mock_open(read_data=ubuntu_sources_deb822_with_comments) + mopen.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + with ( + patch('builtins.open', new=mopen), + patch.object(apt.logger, 'debug') as debug, + ): + repository_mapping.load_deb822("FILENAME") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + ] + debug.assert_called_once_with( + ANY, + 1, # number of errors + "Missing key 'Types' for entry starting on line 11 in FILENAME." + ) + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From 246bd564c58aee8824425057d9f2568b81964037 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 17:06:34 +1300 Subject: [PATCH 14/50] tests: add a unit test for initialiasing RepositoryMapping with deb822 --- tests/unit/test_apt.py | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index 834ac0b6..f7586b3b 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -207,6 +207,13 @@ ## disable security updates while we're at it """ +ubuntu_sources_one_line = """ +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse +""" + class TestApt(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") @@ -615,6 +622,50 @@ def test_load_deb822_w_comments(self): "Missing key 'Types' for entry starting on line 11 in FILENAME." ) + def test_init_with_deb822(self): + """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. + + They should be equivalent with the sample data being used. + """ + mopen_list = mock_open(read_data=ubuntu_sources_one_line) + mopen_list.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + with ( + patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + 'glob.iglob', + new=(lambda s: ['FILENAME'] if s.endswith('.list') else []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + ), + patch('builtins.open', new=mopen_list), + ): + repository_mapping_list = apt.RepositoryMapping() + + mopen_sources = mock_open(read_data=ubuntu_sources_deb822) + mopen_sources.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + with ( + patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + 'glob.iglob', + new=(lambda s: ['FILENAME'] if s.endswith('.sources') else []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + ), + patch('builtins.open', new=mopen_sources), + ): + repository_mapping_sources = apt.RepositoryMapping() + + list_keys = sorted(repository_mapping_list._repository_map.keys()) + sources_keys = sorted(repository_mapping_sources._repository_map.keys()) + assert (sources_keys == list_keys) + + for list_key, sources_key in zip(list_keys, sources_keys): + list_repo = repository_mapping_list[list_key] + sources_repo = repository_mapping_sources[sources_key] + assert list_repo.enabled == sources_repo.enabled + assert list_repo.repotype == sources_repo.repotype + assert list_repo.uri == sources_repo.uri + assert list_repo.release == sources_repo.release + assert list_repo.groups == sources_repo.groups + assert list_repo.gpg_key == sources_repo.gpg_key + assert list_repo.options == sources_repo.options # pyright: ignore[reportUnknownMemberType] + class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") From 87176526f74d51e6a5c882e18efea92950bea93e Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 17:10:25 +1300 Subject: [PATCH 15/50] style: tox -e fmt tests/unit/test_apt.py --- tests/unit/test_apt.py | 143 +++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 61 deletions(-) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index f7586b3b..eb239619 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -3,7 +3,7 @@ import subprocess import unittest -from unittest.mock import mock_open, patch, ANY +from unittest.mock import ANY, mock_open, patch from charms.operator_libs_linux.v0 import apt @@ -463,66 +463,60 @@ def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): def test_get_deb822_options_ubuntu_sources(self): lines = ubuntu_sources_deb822.strip().split("\n") paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - opts = [ - apt.RepositoryMapping._get_deb822_options(p) - for p in paras - ] + opts = [apt.RepositoryMapping._get_deb822_options(p) for p in paras] opts_0, opts_1 = opts opts_0_options, opts_0_line_numbers = opts_0 opts_1_options, opts_1_line_numbers = opts_1 assert opts_0_options == { - 'Types': 'deb', - 'URIs': 'http://nz.archive.ubuntu.com/ubuntu/', - 'Components': 'main restricted universe multiverse', - 'Suites': 'noble noble-updates noble-backports', - 'Signed-By': '/usr/share/keyrings/ubuntu-archive-keyring.gpg', + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Components": "main restricted universe multiverse", + "Suites": "noble noble-updates noble-backports", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", } assert opts_0_line_numbers == { - 'Types': 0, - 'URIs': 1, - 'Suites': 2, - 'Components': 3, - 'Signed-By': 4, + "Types": 0, + "URIs": 1, + "Suites": 2, + "Components": 3, + "Signed-By": 4, } assert opts_1_options == { - 'Types': 'deb', - 'URIs': 'http://security.ubuntu.com/ubuntu', - 'Components': 'main restricted universe multiverse', - 'Suites': 'noble-security', - 'Signed-By': '/usr/share/keyrings/ubuntu-archive-keyring.gpg', + "Types": "deb", + "URIs": "http://security.ubuntu.com/ubuntu", + "Components": "main restricted universe multiverse", + "Suites": "noble-security", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", } assert opts_1_line_numbers == { - 'Types': 6, - 'URIs': 7, - 'Suites': 8, - 'Components': 9, - 'Signed-By': 10, + "Types": 6, + "URIs": 7, + "Suites": 8, + "Components": 9, + "Signed-By": 10, } def test_get_deb822_options_w_comments(self): lines = ubuntu_sources_deb822_with_comments.strip().split("\n") paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - opts = [ - apt.RepositoryMapping._get_deb822_options(p) - for p in paras - ] + opts = [apt.RepositoryMapping._get_deb822_options(p) for p in paras] opts_0, opts_1 = opts opts_0_options, opts_0_line_numbers = opts_0 opts_1_options, opts_1_line_numbers = opts_1 assert opts_0_options == { - 'Components': 'main restricted universe multiverse', - 'Types': 'deb', - 'URIs': 'http://nz.archive.ubuntu.com/ubuntu/', - 'Suites': 'noble noble-updates noble-backports', + "Components": "main restricted universe multiverse", + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Suites": "noble noble-updates noble-backports", } assert opts_0_line_numbers == { - 'Components': 0, - 'Types': 1, - 'URIs': 2, - 'Suites': 8, + "Components": 0, + "Types": 1, + "URIs": 2, + "Suites": 8, } - assert opts_1_options == {'Foo': 'Bar'} - assert opts_1_line_numbers == {'Foo': 10} + assert opts_1_options == {"Foo": "Bar"} + assert opts_1_line_numbers == {"Foo": 10} def test_parse_deb822_paragraph_ubuntu_sources(self): lines = ubuntu_sources_deb822.strip().split("\n") @@ -581,14 +575,20 @@ def test_parse_deb822_lines_w_comments(self): def test_load_deb822_ubuntu_sources(self): with ( - patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch('glob.iglob', new=lambda s: []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "os.path.isfile", new=lambda f: False + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "glob.iglob", new=lambda s: [] + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] ): repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map mopen = mock_open(read_data=ubuntu_sources_deb822) - mopen.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - with patch('builtins.open', new=mopen): + mopen.return_value.__iter__ = lambda self: iter( + self.readline, "" + ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + with patch("builtins.open", new=mopen): repository_mapping.load_deb822("") assert sorted(repository_mapping._repository_map.keys()) == [ "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", @@ -599,16 +599,22 @@ def test_load_deb822_ubuntu_sources(self): def test_load_deb822_w_comments(self): with ( - patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch('glob.iglob', new=lambda s: []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "os.path.isfile", new=lambda f: False + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "glob.iglob", new=lambda s: [] + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] ): repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map mopen = mock_open(read_data=ubuntu_sources_deb822_with_comments) - mopen.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + mopen.return_value.__iter__ = lambda self: iter( + self.readline, "" + ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] with ( - patch('builtins.open', new=mopen), - patch.object(apt.logger, 'debug') as debug, + patch("builtins.open", new=mopen), + patch.object(apt.logger, "debug") as debug, ): repository_mapping.load_deb822("FILENAME") assert sorted(repository_mapping._repository_map.keys()) == [ @@ -619,7 +625,7 @@ def test_load_deb822_w_comments(self): debug.assert_called_once_with( ANY, 1, # number of errors - "Missing key 'Types' for entry starting on line 11 in FILENAME." + "Missing key 'Types' for entry starting on line 11 in FILENAME.", ) def test_init_with_deb822(self): @@ -628,32 +634,44 @@ def test_init_with_deb822(self): They should be equivalent with the sample data being used. """ mopen_list = mock_open(read_data=ubuntu_sources_one_line) - mopen_list.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + mopen_list.return_value.__iter__ = lambda self: iter( + self.readline, "" + ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] with ( - patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] patch( - 'glob.iglob', - new=(lambda s: ['FILENAME'] if s.endswith('.list') else []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + "os.path.isfile", new=lambda f: False + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "glob.iglob", + new=( + lambda s: ["FILENAME"] if s.endswith(".list") else [] + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] ), - patch('builtins.open', new=mopen_list), + patch("builtins.open", new=mopen_list), ): repository_mapping_list = apt.RepositoryMapping() mopen_sources = mock_open(read_data=ubuntu_sources_deb822) - mopen_sources.return_value.__iter__ = lambda self: iter(self.readline, '') # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + mopen_sources.return_value.__iter__ = lambda self: iter( + self.readline, "" + ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] with ( - patch('os.path.isfile', new=lambda f: False), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] patch( - 'glob.iglob', - new=(lambda s: ['FILENAME'] if s.endswith('.sources') else []), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] + "os.path.isfile", new=lambda f: False + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] + patch( + "glob.iglob", + new=( + lambda s: ["FILENAME"] if s.endswith(".sources") else [] + ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] ), - patch('builtins.open', new=mopen_sources), + patch("builtins.open", new=mopen_sources), ): repository_mapping_sources = apt.RepositoryMapping() list_keys = sorted(repository_mapping_list._repository_map.keys()) sources_keys = sorted(repository_mapping_sources._repository_map.keys()) - assert (sources_keys == list_keys) + assert sources_keys == list_keys for list_key, sources_key in zip(list_keys, sources_keys): list_repo = repository_mapping_list[list_key] @@ -664,7 +682,10 @@ def test_init_with_deb822(self): assert list_repo.release == sources_repo.release assert list_repo.groups == sources_repo.groups assert list_repo.gpg_key == sources_repo.gpg_key - assert list_repo.options == sources_repo.options # pyright: ignore[reportUnknownMemberType] + assert ( + list_repo.options # pyright: ignore[reportUnknownMemberType] + == sources_repo.options # pyright: ignore[reportUnknownMemberType] + ) class TestAptBareMethods(unittest.TestCase): From 9dfadc16619fb468f629f3e8a718e2353de37bdf Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 17:13:28 +1300 Subject: [PATCH 16/50] fix: correct docstring --- lib/charms/operator_libs_linux/v0/apt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 378ee8cb..1cc88194 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1335,10 +1335,10 @@ def load_deb822(self, filename: str) -> None: /etc/apt/sources.list.d/ubuntu.sources The semantics of `load_deb822` slightly different to `load`: - `load` calls `_parse`, with reads a commented out line as an entry that is not enabled + `load` calls `_parse`, which reads a commented out line as an entry that is not enabled `load_deb822` strips out comments entirely when parsing a file into paragraphs, and assumes that comments have been removed when parsing individual paragraphs/entry, - instead reading the 'Enabled' key to determine if an entry is enabled + instead only reading the 'Enabled' key to determine if an entry is enabled """ with open(filename, "r") as f: repos, errors = self._parse_deb822_lines(f, filename=filename) From c44a4a5e2384a6b9891d4379fc356c84bd8d8a45 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 21 Nov 2024 17:34:43 +1300 Subject: [PATCH 17/50] fix: python3.8 context managers and clean up linter directives --- tests/unit/test_apt.py | 117 ++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 66 deletions(-) diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index eb239619..316cb29b 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -1,7 +1,10 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. +# pyright: reportPrivateUsage=false + import subprocess +import typing import unittest from unittest.mock import ANY, mock_open, patch @@ -574,21 +577,18 @@ def test_parse_deb822_lines_w_comments(self): assert isinstance(error, apt.InvalidSourceError) def test_load_deb822_ubuntu_sources(self): - with ( - patch( - "os.path.isfile", new=lambda f: False - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch( - "glob.iglob", new=lambda s: [] - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - ): - repository_mapping = apt.RepositoryMapping() + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map - mopen = mock_open(read_data=ubuntu_sources_deb822) - mopen.return_value.__iter__ = lambda self: iter( - self.readline, "" - ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - with patch("builtins.open", new=mopen): + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): repository_mapping.load_deb822("") assert sorted(repository_mapping._repository_map.keys()) == [ "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", @@ -598,25 +598,22 @@ def test_load_deb822_ubuntu_sources(self): ] def test_load_deb822_w_comments(self): - with ( - patch( - "os.path.isfile", new=lambda f: False - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch( - "glob.iglob", new=lambda s: [] - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - ): - repository_mapping = apt.RepositoryMapping() + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map - mopen = mock_open(read_data=ubuntu_sources_deb822_with_comments) - mopen.return_value.__iter__ = lambda self: iter( - self.readline, "" - ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - with ( - patch("builtins.open", new=mopen), - patch.object(apt.logger, "debug") as debug, + + with patch( + "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments ): - repository_mapping.load_deb822("FILENAME") + with patch.object(apt.logger, "debug") as debug: + repository_mapping.load_deb822("FILENAME") assert sorted(repository_mapping._repository_map.keys()) == [ "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", @@ -633,41 +630,29 @@ def test_init_with_deb822(self): They should be equivalent with the sample data being used. """ - mopen_list = mock_open(read_data=ubuntu_sources_one_line) - mopen_list.return_value.__iter__ = lambda self: iter( - self.readline, "" - ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - with ( - patch( - "os.path.isfile", new=lambda f: False - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch( - "glob.iglob", - new=( - lambda s: ["FILENAME"] if s.endswith(".list") else [] - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - ), - patch("builtins.open", new=mopen_list), - ): - repository_mapping_list = apt.RepositoryMapping() - - mopen_sources = mock_open(read_data=ubuntu_sources_deb822) - mopen_sources.return_value.__iter__ = lambda self: iter( - self.readline, "" - ) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - with ( - patch( - "os.path.isfile", new=lambda f: False - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType] - patch( - "glob.iglob", - new=( - lambda s: ["FILENAME"] if s.endswith(".sources") else [] - ), # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType, reportUnknownMemberType] - ), - patch("builtins.open", new=mopen_sources), - ): - repository_mapping_sources = apt.RepositoryMapping() + + def isnt_file(f: str) -> bool: + return False + + def iglob_list(s: str) -> typing.Iterable[str]: + if s.endswith(".list"): + return ["FILENAME"] + return [] + + def iglob_sources(s: str) -> typing.Iterable[str]: + if s.endswith(".sources"): + return ["FILENAME"] + return [] + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_list): + repository_mapping_list = apt.RepositoryMapping() + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_sources): + repository_mapping_sources = apt.RepositoryMapping() list_keys = sorted(repository_mapping_list._repository_map.keys()) sources_keys = sorted(repository_mapping_sources._repository_map.keys()) From 2df185e165eb50b5bfb88a2e167a492b22606599 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 27 Nov 2024 15:18:35 +1300 Subject: [PATCH 18/50] feat: make DebianRepository aware of deb822 format for some operations add: add ability to write a deb822 format file disable: raise NotImplementedError for deb822 format files gpg_key: use existing import_key functionality to provide keys specified in the stanza itself as a file for compatibility Also refactor Deb822 functionality to a separate class. Also move deb822 unit tests to the more appropriate test_repo.py --- lib/charms/operator_libs_linux/v0/apt.py | 446 +++++++++++++++-------- tests/unit/test_apt.py | 291 +-------------- tests/unit/test_repo.py | 298 ++++++++++++++- 3 files changed, 595 insertions(+), 440 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 1cc88194..bb55c758 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -107,10 +107,10 @@ import os import re import subprocess -from collections.abc import Mapping +import textwrap from enum import Enum from subprocess import PIPE, CalledProcessError, check_output -from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Tuple, Union from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -923,7 +923,8 @@ def __init__( groups: List[str], filename: Optional[str] = "", gpg_key_filename: Optional[str] = "", - options: Optional[dict] = None, + options: Optional[Dict[str, str]] = None, + deb822_stanza: Optional["_Deb822Stanza"] = None, ): self._enabled = enabled self._repotype = repotype @@ -933,6 +934,7 @@ def __init__( self._filename = filename self._gpg_key_filename = gpg_key_filename self._options = options + self._deb822_stanza = deb822_stanza @property def enabled(self): @@ -971,14 +973,15 @@ def filename(self, fname: str) -> None: Args: fname: a filename to write the repository information to. """ - if not fname.endswith(".list"): - raise InvalidSourceError("apt source filenames should end in .list!") - + if not fname.endswith(".list") or fname.endswith(".sources"): + raise InvalidSourceError("apt source filenames should end in .list or .sources!") self._filename = fname @property def gpg_key(self): """Returns the path to the GPG key for this repository.""" + if not self._gpg_key_filename and self._deb822_stanza is not None: + self._gpg_key_filename = self._deb822_stanza.get_gpg_key_filename() return self._gpg_key_filename @property @@ -987,7 +990,7 @@ def options(self): return self._options def make_options_string(self) -> str: - """Generate the complete options string for a a repository. + """Generate the complete one-line-style options string for a repository. Combining `gpg_key`, if set, and the rest of the options to find a complex repo string. @@ -1013,7 +1016,7 @@ def prefix_from_uri(uri: str) -> str: @staticmethod def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "DebianRepository": - """Instantiate a new `DebianRepository` a `sources.list` entry line. + """Instantiate a new `DebianRepository` from a `sources.list` entry line. Args: repo_line: a string representing a repository entry @@ -1050,10 +1053,41 @@ def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "Debian return repo def disable(self) -> None: - """Remove this repository from consideration. + """Remove this repository by disabling it in the source file. + + WARNING: This method does NOT alter the `self.enabled` flag. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza. Raises a NotImplementedError in this case. - Disable it instead of removing from the repository file. + Potential semantics for disable on a deb822 defined repository are + described below. Please open an issue if you require the ability to + disable deb822 defined repositories. + + Multiple DebianRepository objects may be defined by the same stanza. + If the stanza that defines this object defines other DebianRepository + objects, an error will be raised on attempting to disable this repository, + unless an `always_disable_deb822_stanza` is passed, in which case the entire + stanza should be disabled -- which will also disable all other repositories + defined by the same stanza. """ + if self._deb822_stanza is not None: + raise NotImplementedError( + textwrap.dedent( + """ + Disabling a repository defined by a deb822 format source is not implemented. + Please raise an issue if you require this feature. + This repository is: + {str_self} + Defined by the deb822 stanza: + {str_stanza} + ) + """.format( + str_self=textwrap.indent(str(self), " " * 2), + str_stanza=textwrap.indent(str(self._deb822_stanza), " " * 2), + ) + ).strip() + ) searcher = "{} {}{} {}".format( self.repotype, self.make_options_string(), self.uri, self.release ) @@ -1182,7 +1216,7 @@ def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: keyf.write(key_material) -class RepositoryMapping(Mapping): +class RepositoryMapping(Mapping[str, DebianRepository]): """An representation of known repositories. Instantiation of `RepositoryMapping` will iterate through the @@ -1200,22 +1234,30 @@ class RepositoryMapping(Mapping): def __init__(self): self._repository_map: Dict[str, DebianRepository] = {} - # Repositories that we're adding -- used to implement mode param self.default_file = "/etc/apt/sources.list" + self.default_sources = "/etc/apt/sources.list.d/ubuntu.sources" # read sources.list if it exists + # ignore InvalidSourceError if ubuntu.sources also exists + # -- in this case, sources.list just contains a comment if os.path.isfile(self.default_file): - self.load(self.default_file) + try: + self.load(self.default_file) + except InvalidSourceError: + if not os.path.isfile(self.default_sources): + raise # read sources.list.d for file in glob.iglob("/etc/apt/sources.list.d/*.list"): self.load(file) - for file in glob.iglob("/etc/apt/sources.list.d/*.sources"): self.load_deb822(file) - def __contains__(self, key: str) -> bool: - """Magic method for checking presence of repo in mapping.""" + def __contains__(self, key: Any) -> bool: + """Magic method for checking presence of repo in mapping. + + Checks against the string names used to identify repositories. + """ return key in self._repository_map def __len__(self) -> int: @@ -1223,7 +1265,10 @@ def __len__(self) -> int: return len(self._repository_map) def __iter__(self) -> Iterable[DebianRepository]: - """Return iterator for RepositoryMapping.""" + """Return iterator for RepositoryMapping. + + Iterates over the DebianRepository values rather than the string names. + """ return iter(self._repository_map.values()) def __getitem__(self, repository_uri: str) -> DebianRepository: @@ -1234,6 +1279,62 @@ def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None """Add a `DebianRepository` to the cache.""" self._repository_map[repository_uri] = repository + def load_deb822(self, filename: str) -> None: + """Load a deb822 format repository source file into the cache. + + In contrast to one-line-style, the deb822 format specifies a repository + using a multi-line paragraph. Paragraphs are separated by whitespace, + and each definition consists of lines that are either key: value pairs, + or continuations of the previous value. + + Read more about the deb822 format here: + https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html + For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: + /etc/apt/sources.list.d/ubuntu.sources + """ + with open(filename, "r") as f: + repos, errors = self._parse_deb822_lines(f, filename=filename) + + for repo in repos: + repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) + self._repository_map[repo_identifier] = repo + + if errors: + logger.debug( + "the following %d error(s) were encountered when reading deb822 sources:\n%s", + len(errors), + "\n".join(str(e) for e in errors), + ) + + if repos: + logger.info("parsed %d apt package repositories", len(repos)) + else: + raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) + + @classmethod + def _parse_deb822_lines( + cls, + lines: Iterable[str], + filename: str = "", + ) -> Tuple[List[DebianRepository], List[InvalidSourceError]]: + """Parse lines from a deb822 file into a list of repos and a list of errors. + + The semantics of `_parse_deb822_lines` slightly different to `_parse`: + `_parse` reads a commented out line as an entry that is not enabled + `_parse_deb822_lines` strips out comments entirely when parsing a file into stanzas, + instead only reading the 'Enabled' key to determine if an entry is enabled + """ + repositories: List[DebianRepository] = [] + errors: List[InvalidSourceError] = [] + for numbered_lines in _iter_deb822_stanzas(lines): + try: + stanza = _Deb822Stanza(numbered_lines=numbered_lines, filename=filename) + except InvalidSourceError as e: + errors.append(e) + else: + repositories.extend(stanza.repositories) + return repositories, errors + def load(self, filename: str): """Load a one-line-style format repository source file into the cache. @@ -1318,102 +1419,144 @@ def _parse(line: str, filename: str) -> DebianRepository: else: raise InvalidSourceError("An invalid sources line was found in %s!", filename) - def load_deb822(self, filename: str) -> None: - """Load a deb822 format repository source file into the cache. + def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: + """Add a new repository to the system. May have destructive or surprising behaviour. + + Will write in the deb822 format if: + 1. repo.filename has the '.sources' extension + 2. repo.filename is unset (""), but /etc/apt/sources.list.d/ubuntu.sources exists + Otherwise the one-per-line style will be used. + + WARNING: the default_filename keyword argument is provided for backwards compatibility + only. It is not used, and was not used in the previous revision of this library. + + WARNING: in the one-per-line format case, will mutate repo.options to add the 'signed-by' + key if both options and gpg_file are truthy (for example if a gpg_key_filename and + non-empty options were provided at DebianRepository initialisation time). If options were + not provided or are empty, but a gpg_key is available, will silently fail to include it. + + WARNING: if repo.filename is falsey, the new filename is calculated only from the repo's + uri and release, but repos are assumed to be uniquely identified by the combination of + repotype, uri, and release. Adding two repos with differing repotypes but identical uris + and releases will result in the second repo clobbering the file written by the first. + In this case, repo.filename must be set appropriately to avoid data loss. + + WARNING: if repo.filename is truthy, and that file exists, this method will clobber that + file with a single entry for the repo being added. Set it to a falsey value to have a new + filename derived from its uri and release (which will also clobber any existing file), or + set the filename that you want to write to -- or construct a new DebianRepository object + with the filename you want to write to. + For example: repo.filename = my_filename + For example: DebianRepository(uri=repo.uri, filename=my_filename, ...) + + WARNING: if repo.enabled is falsey, the repository will be added in the disabled state. + Note that repo.disable() does not affect this value. If repo.enabled does not match the + value you expect, construct a new DebianRepository object with the appropriate value. + For example: DebianRepository(uri=repo.uri, enabled=my_value, ...) + """ + if repo.filename: + fname = repo.filename + else: + fname = "{}-{}.{}".format( + DebianRepository.prefix_from_uri(repo.uri), + repo.release.replace("/", "-"), + "sources" if os.path.isfile(self.default_sources) else "list", + ) - Args: - filename: the path to the repository file + _, fname_extension = fname.rsplit(".", maxsplit=1) + if fname_extension == "sources": + text = "\n".join( + "{}: {}".format(key, value) + for key, value in _opts_from_repository( + repo, + stanza=repo._deb822_stanza, # pyright: ignore[reportPrivateUsage] + ) + ) + else: + options = repo.options if repo.options else {} + if repo.gpg_key: + options["signed-by"] = repo.gpg_key + text = ( + "{}".format("#" if not repo.enabled else "") + + "{} {}{} ".format(repo.repotype, repo.make_options_string(), repo.uri) + + "{} {}\n".format(repo.release, " ".join(repo.groups)) + ) - In contrast to one-line-style, the deb822 format specifies a repository - using a multi-line paragraph. Paragraphs are separated by whitespace, - and each definition consists of lines that are either key: value pairs, - or continuations of the previous value. + with open(fname, "wb") as f: + f.write(text.encode("utf-8")) - Read more about the deb822 format here: - https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html - For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: - /etc/apt/sources.list.d/ubuntu.sources + self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo - The semantics of `load_deb822` slightly different to `load`: - `load` calls `_parse`, which reads a commented out line as an entry that is not enabled - `load_deb822` strips out comments entirely when parsing a file into paragraphs, and - assumes that comments have been removed when parsing individual paragraphs/entry, - instead only reading the 'Enabled' key to determine if an entry is enabled + def disable(self, repo: DebianRepository) -> None: + """Remove a repository by disabling it in the source file. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza, and will raise a NotImplementedError if called on one. + + WARNING: This method does NOT alter the `.enabled` flag on the DebianRepository. """ - with open(filename, "r") as f: - repos, errors = self._parse_deb822_lines(f, filename=filename) + repo.disable() - for repo in repos: - repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) - self._repository_map[repo_identifier] = repo - if errors: - logger.debug( - "the following %d error(s) were encountered when reading deb822 format sources:\n%s", - len(errors), - "\n".join(str(e) for e in errors), - ) +class _Deb822Stanza: + """Representation of a stanza from a deb822 source file. - if repos: - logger.info("parsed %d apt package repositories", len(repos)) - else: - raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) + May define multiple DebianRepository objects. + """ - @classmethod - def _parse_deb822_lines( - cls, - lines: Iterable[str], - filename: str = "", - ) -> Tuple[List[DebianRepository], List[InvalidSourceError]]: - """Parse lines from a deb822 file into a list of repos and a list of errors.""" - repositories: List[DebianRepository] = [] - errors: List[InvalidSourceError] = [] - for paragraph in cls._iter_deb822_paragraphs(lines): - try: - repos = cls._parse_deb822_paragraph(paragraph, filename=filename) - except InvalidSourceError as e: - errors.append(e) - else: - repositories.extend(repos) - return repositories, errors + def __init__(self, numbered_lines: List[Tuple[int, str]], filename: str = ""): + self._numbered_lines = numbered_lines + self._filename = filename + self._gpg_key_from_stanza: Optional[str] = None # may be set in _parse_stanzas + self._gpg_key_filename: Optional[str] = None # may be set in _parse_stanzas + self._repositories = self._parse_stanza(lines=numbered_lines, filename=filename) - @staticmethod - def _iter_deb822_paragraphs(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: - """Given lines from a deb822 format file, yield paragraphs. + def get_gpg_key_filename(self) -> str: + """Return the path to the GPG key for this repository. - A paragraph is a list of numbered lines that make up a source entry, - with comments stripped out (but accounted for in line numbering). + Import the key first, if the key itself was provided in the stanza. """ - current_paragraph: List[Tuple[int, str]] = [] - for n, line in enumerate(lines): # 0 indexed line numbers, following `load` - if not line.strip(): # blank lines separate paragraphs - if current_paragraph: - yield current_paragraph - current_paragraph = [] - continue - content, _delim, _comment = line.partition("#") - if content.strip(): # skip (potentially indented) comment line - current_paragraph.append((n, content.rstrip())) # preserve indent - if current_paragraph: - yield current_paragraph + if self._gpg_key_filename is not None: + return self._gpg_key_filename + if self._gpg_key_from_stanza is None: + return "" + # a gpg key was provided in the stanza + # and we haven't already imported it + self._gpg_key_filename = import_key(self._gpg_key_from_stanza) + return self._gpg_key_filename - @classmethod - def _parse_deb822_paragraph( - cls, - lines: List[Tuple[int, str]], - filename: str = "", - ) -> List[DebianRepository]: - """Parse a list of numbered lines forming a deb822 style repository definition. + @property + def repositories(self) -> Tuple[DebianRepository, ...]: + """The repositories defined by this deb822 stanza.""" + return self._repositories + + def __str__(self) -> str: + """Return formatted representation of object instantiation.""" + return textwrap.dedent( + """ + {name}( + filename={filename}, + numbered_lines=[ + {lines} + ], + ) + """.format( + name=self.__class__.__name__, + filename=self._filename, + lines="\n ".join(str(line) for line in self._numbered_lines), + ) + ).strip() - Args: - lines: a list of numbered lines forming a deb822 paragraph - filename: the name of the file being read (for DebianRepository and errors) + def _parse_stanza( + self, lines: List[Tuple[int, str]], filename: str = "" + ) -> Tuple[DebianRepository, ...]: + """Return a list of DebianRepository objects defined by this deb822 stanza. Raises: InvalidSourceError if the source type is unknown or contains malformed entries """ - options, line_numbers = cls._get_deb822_options(lines) - + options, line_numbers = self._get_options(lines) + # Enabled enabled_field = options.pop("Enabled", "yes") if enabled_field == "yes": enabled = True @@ -1431,9 +1574,13 @@ def _parse_deb822_paragraph( file=filename, ) ) - - gpg_key = options.pop("Signed-By", "") - + # Signed-By + gpg_key_file = options.pop("Signed-By", "") + if "\n" in gpg_key_file: + # actually a literal multi-line gpg-key rather than a filename + self._gpg_key_from_stanza = gpg_key_file + gpg_key_file = "" + # Types try: repotypes = options.pop("Types").split() uris = options.pop("URIs").split() @@ -1447,7 +1594,7 @@ def _parse_deb822_paragraph( file=filename, ) ) - + # Components components: List[str] if len(suites) == 1 and suites[0].endswith("/"): if "Components" in options: @@ -1480,8 +1627,7 @@ def _parse_deb822_paragraph( ) ) components = options.pop("Components").split() - - return [ + return tuple( DebianRepository( enabled=enabled, repotype=repotype, @@ -1489,15 +1635,16 @@ def _parse_deb822_paragraph( release=suite, groups=components, filename=filename, - gpg_key_filename=gpg_key, # TODO: gpg_key can be a literal key, not just a filename + gpg_key_filename=gpg_key_file, options=options, + deb822_stanza=self, ) for repotype, uri, suite in itertools.product(repotypes, uris, suites) - ] + ) @staticmethod - def _get_deb822_options( - lines: Iterable[Tuple[int, str]] + def _get_options( + lines: Iterable[Tuple[int, str]], ) -> Tuple[Dict[str, str], Dict[str, int]]: parts: Dict[str, List[str]] = {} line_numbers: Dict[str, int] = {} @@ -1515,48 +1662,51 @@ def _get_deb822_options( options = {k: "\n".join(v) for k, v in parts.items()} return options, line_numbers - def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: - """Add a new repository to the system. - - Args: - repo: a `DebianRepository` object - default_filename: an (Optional) filename if the default is not desirable - """ - new_filename = "{}-{}.list".format( - DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") - ) - - fname = repo.filename or new_filename - - options = repo.options if repo.options else {} - if repo.gpg_key: - options["signed-by"] = repo.gpg_key - - with open(fname, "wb") as f: - f.write( - ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, repo.make_options_string(), repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) - ).encode("utf-8") - ) - - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo - - def disable(self, repo: DebianRepository) -> None: - """Remove a repository. Disable by default. - - Args: - repo: a `DebianRepository` to disable - """ - searcher = "{} {}{} {}".format( - repo.repotype, repo.make_options_string(), repo.uri, repo.release - ) - for line in fileinput.input(repo.filename, inplace=True): - if re.match(r"^{}\s".format(re.escape(searcher)), line): - print("# {}".format(line), end="") - else: - print(line, end="") +def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: + """Given lines from a deb822 format file, yield a stanza of lines. - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo + A stanza is a list of numbered lines that make up a source entry, + with comments stripped out (but accounted for in line numbering). + """ + current_paragraph: List[Tuple[int, str]] = [] + for n, line in enumerate(lines): # 0 indexed line numbers, following `load` + if not line.strip(): # blank lines separate paragraphs + if current_paragraph: + yield current_paragraph + current_paragraph = [] + continue + content, _delim, _comment = line.partition("#") + if content.strip(): # skip (potentially indented) comment line + current_paragraph.append((n, content.rstrip())) # preserve indent + if current_paragraph: + yield current_paragraph + + +def _opts_from_repository( + repo: DebianRepository, stanza: Optional[_Deb822Stanza] = None +) -> Dict[str, str]: + """Return repo information as deb822 format keys and values.""" + opts = { + "Enabled": ("yes" if repo.enabled else "no"), + "Types": repo.repotype, + "URIs": repo.uri, + "Suites": repo.release, + } + # Signed-By + # use the actual gpg key if it was provided in the stanza + # otherwise use the filename if that was provided + gpg_key: Optional[str] = None + if stanza is not None: + gpg_key = stanza._gpg_key_from_stanza + if gpg_key is None: # no inline gpg key provided + gpg_key = repo.gpg_key + if gpg_key: + opts["Signed-By"] = gpg_key + # Components + if repo.groups: + opts["Components"] = " ".join(repo.groups) + # options + if repo.options is not None: + opts.update(repo.options) + return opts diff --git a/tests/unit/test_apt.py b/tests/unit/test_apt.py index 316cb29b..99c43ca5 100644 --- a/tests/unit/test_apt.py +++ b/tests/unit/test_apt.py @@ -1,12 +1,9 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. -# pyright: reportPrivateUsage=false - import subprocess -import typing import unittest -from unittest.mock import ANY, mock_open, patch +from unittest.mock import patch from charms.operator_libs_linux.v0 import apt @@ -175,48 +172,6 @@ Description-md5: e7f99df3aa92cf870d335784e155ec33 """ -ubuntu_sources_deb822 = """ -Types: deb -URIs: http://nz.archive.ubuntu.com/ubuntu/ -Suites: noble noble-updates noble-backports -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg - -Types: deb -URIs: http://security.ubuntu.com/ubuntu -Suites: noble-security -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -""" - -ubuntu_sources_deb822_with_comments = """ -Components: main restricted universe multiverse # this lib doesn't care about order -Types: deb # this could include deb-src as well or instead -URIs: http://nz.archive.ubuntu.com/ubuntu/ - # there can be multiple space separated URIs - # sources are specified in priority order - # apt does some de-duplication of sources after parsing too -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -# let's make this insecure! (jk, just testing parsing) -Suites: noble noble-updates noble-backports - -Foo: Bar # this is a separate (malformed) entry - -#Types: deb -#URIs: http://security.ubuntu.com/ubuntu -#Suites: noble-security -#Components: main restricted universe multiverse -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -## disable security updates while we're at it -""" - -ubuntu_sources_one_line = """ -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse -""" - class TestApt(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") @@ -428,250 +383,6 @@ def test_can_parse_epoch_and_version(self): ("2", "9.8-7ubuntu6"), apt.DebianPackage._get_epoch_from_version("2:9.8-7ubuntu6") ) - def test_iter_deb822_paragraphs_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - assert paras == [ - [ - (0, "Types: deb"), - (1, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), - (2, "Suites: noble noble-updates noble-backports"), - (3, "Components: main restricted universe multiverse"), - (4, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), - ], - [ - (6, "Types: deb"), - (7, "URIs: http://security.ubuntu.com/ubuntu"), - (8, "Suites: noble-security"), - (9, "Components: main restricted universe multiverse"), - (10, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), - ], - ] - - def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - assert paras == [ - [ - (0, "Components: main restricted universe multiverse"), - (1, "Types: deb"), - (2, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), - (8, "Suites: noble noble-updates noble-backports"), - ], - [ - (10, "Foo: Bar"), - ], - ] - - def test_get_deb822_options_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - opts = [apt.RepositoryMapping._get_deb822_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, opts_0_line_numbers = opts_0 - opts_1_options, opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Components": "main restricted universe multiverse", - "Suites": "noble noble-updates noble-backports", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - assert opts_0_line_numbers == { - "Types": 0, - "URIs": 1, - "Suites": 2, - "Components": 3, - "Signed-By": 4, - } - assert opts_1_options == { - "Types": "deb", - "URIs": "http://security.ubuntu.com/ubuntu", - "Components": "main restricted universe multiverse", - "Suites": "noble-security", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - assert opts_1_line_numbers == { - "Types": 6, - "URIs": 7, - "Suites": 8, - "Components": 9, - "Signed-By": 10, - } - - def test_get_deb822_options_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - paras = list(apt.RepositoryMapping._iter_deb822_paragraphs(lines)) - opts = [apt.RepositoryMapping._get_deb822_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, opts_0_line_numbers = opts_0 - opts_1_options, opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Components": "main restricted universe multiverse", - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Suites": "noble noble-updates noble-backports", - } - assert opts_0_line_numbers == { - "Components": 0, - "Types": 1, - "URIs": 2, - "Suites": 8, - } - assert opts_1_options == {"Foo": "Bar"} - assert opts_1_line_numbers == {"Foo": 10} - - def test_parse_deb822_paragraph_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - main, security = apt.RepositoryMapping._iter_deb822_paragraphs(lines) - repos = apt.RepositoryMapping._parse_deb822_paragraph(main) - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - repos = apt.RepositoryMapping._parse_deb822_paragraph(security) - assert len(repos) == 1 - [repo] = repos - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://security.ubuntu.com/ubuntu" - assert repo.release == "noble-security" - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - - def test_parse_deb822_paragraph_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - ok_para, bad_para = apt.RepositoryMapping._iter_deb822_paragraphs(lines) - repos = apt.RepositoryMapping._parse_deb822_paragraph(ok_para) - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "" - with self.assertRaises(apt.InvalidSourceError): - apt.RepositoryMapping._parse_deb822_paragraph(bad_para) - - def test_parse_deb822_lines_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - print(repos[0].__dict__) - assert len(repos) == 4 - assert not errors - - def test_parse_deb822_lines_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - assert len(repos) == 3 - assert len(errors) == 1 - [error] = errors - assert isinstance(error, apt.InvalidSourceError) - - def test_load_deb822_ubuntu_sources(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - repository_mapping.load_deb822("") - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - "deb-http://security.ubuntu.com/ubuntu-noble-security", - ] - - def test_load_deb822_w_comments(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch( - "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments - ): - with patch.object(apt.logger, "debug") as debug: - repository_mapping.load_deb822("FILENAME") - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - ] - debug.assert_called_once_with( - ANY, - 1, # number of errors - "Missing key 'Types' for entry starting on line 11 in FILENAME.", - ) - - def test_init_with_deb822(self): - """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. - - They should be equivalent with the sample data being used. - """ - - def isnt_file(f: str) -> bool: - return False - - def iglob_list(s: str) -> typing.Iterable[str]: - if s.endswith(".list"): - return ["FILENAME"] - return [] - - def iglob_sources(s: str) -> typing.Iterable[str]: - if s.endswith(".sources"): - return ["FILENAME"] - return [] - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_list): - repository_mapping_list = apt.RepositoryMapping() - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_sources): - repository_mapping_sources = apt.RepositoryMapping() - - list_keys = sorted(repository_mapping_list._repository_map.keys()) - sources_keys = sorted(repository_mapping_sources._repository_map.keys()) - assert sources_keys == list_keys - - for list_key, sources_key in zip(list_keys, sources_keys): - list_repo = repository_mapping_list[list_key] - sources_repo = repository_mapping_sources[sources_key] - assert list_repo.enabled == sources_repo.enabled - assert list_repo.repotype == sources_repo.repotype - assert list_repo.uri == sources_repo.uri - assert list_repo.release == sources_repo.release - assert list_repo.groups == sources_repo.groups - assert list_repo.gpg_key == sources_repo.gpg_key - assert ( - list_repo.options # pyright: ignore[reportUnknownMemberType] - == sources_repo.options # pyright: ignore[reportUnknownMemberType] - ) - class TestAptBareMethods(unittest.TestCase): @patch("charms.operator_libs_linux.v0.apt.check_output") diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index 0233518e..cc17d680 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -1,10 +1,15 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. +# pyright: reportPrivateUsage=false + import os +import typing +import unittest +from unittest.mock import ANY, mock_open, patch from charms.operator_libs_linux.v0 import apt -from pyfakefs.fake_filesystem_unittest import TestCase +from pyfakefs.fake_filesystem_unittest import TestCase as PyFakeFsTestCase sources_list = """## This is a comment which should be ignored! deb http://us.archive.ubuntu.com/ubuntu focal main restricted universe multiverse @@ -30,7 +35,7 @@ """ -class TestRepositoryMapping(TestCase): +class TestRepositoryMapping(PyFakeFsTestCase): def setUp(self): self.setUpPyfakefs() self.fs.create_file("/etc/apt/sources.list", contents=sources_list) @@ -146,3 +151,292 @@ def test_can_add_repositories_from_string_with_options(self): "deb [arch=amd64 signed-by=/foo/gpg.key] https://example.com/foo focal bar baz\n", open(d.filename).readlines(), ) + + +ubuntu_sources_deb822 = """ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +""" + +ubuntu_sources_deb822_with_comments = """ +Components: main restricted universe multiverse # this lib doesn't care about order +Types: deb # this could include deb-src as well or instead +URIs: http://nz.archive.ubuntu.com/ubuntu/ + # there can be multiple space separated URIs + # sources are specified in priority order + # apt does some de-duplication of sources after parsing too +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +# let's make this insecure! (jk, just testing parsing) +Suites: noble noble-updates noble-backports + +Foo: Bar # this is a separate (malformed) entry + +#Types: deb +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +## disable security updates while we're at it +""" + +ubuntu_sources_one_line = """ +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse +""" + + +class TestRepositoryMappingDeb822Behaviour(unittest.TestCase): + def test_iter_deb822_paragraphs_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + assert paras == [ + [ + (0, "Types: deb"), + (1, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), + (2, "Suites: noble noble-updates noble-backports"), + (3, "Components: main restricted universe multiverse"), + (4, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), + ], + [ + (6, "Types: deb"), + (7, "URIs: http://security.ubuntu.com/ubuntu"), + (8, "Suites: noble-security"), + (9, "Components: main restricted universe multiverse"), + (10, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), + ], + ] + + def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + assert paras == [ + [ + (0, "Components: main restricted universe multiverse"), + (1, "Types: deb"), + (2, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), + (8, "Suites: noble noble-updates noble-backports"), + ], + [ + (10, "Foo: Bar"), + ], + ] + + def test_get_deb822_options_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + opts = [apt._Deb822Stanza._get_options(p) for p in paras] + opts_0, opts_1 = opts + opts_0_options, opts_0_line_numbers = opts_0 + opts_1_options, opts_1_line_numbers = opts_1 + assert opts_0_options == { + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Components": "main restricted universe multiverse", + "Suites": "noble noble-updates noble-backports", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", + } + assert opts_0_line_numbers == { + "Types": 0, + "URIs": 1, + "Suites": 2, + "Components": 3, + "Signed-By": 4, + } + assert opts_1_options == { + "Types": "deb", + "URIs": "http://security.ubuntu.com/ubuntu", + "Components": "main restricted universe multiverse", + "Suites": "noble-security", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", + } + assert opts_1_line_numbers == { + "Types": 6, + "URIs": 7, + "Suites": 8, + "Components": 9, + "Signed-By": 10, + } + + def test_get_deb822_options_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + opts = [apt._Deb822Stanza._get_options(p) for p in paras] + opts_0, opts_1 = opts + opts_0_options, opts_0_line_numbers = opts_0 + opts_1_options, opts_1_line_numbers = opts_1 + assert opts_0_options == { + "Components": "main restricted universe multiverse", + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Suites": "noble noble-updates noble-backports", + } + assert opts_0_line_numbers == { + "Components": 0, + "Types": 1, + "URIs": 2, + "Suites": 8, + } + assert opts_1_options == {"Foo": "Bar"} + assert opts_1_line_numbers == {"Foo": 10} + + def test_parse_deb822_paragraph_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + main, security = apt._iter_deb822_stanzas(lines) + repos = apt._Deb822Stanza(main).repositories + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + repos = apt._Deb822Stanza(security).repositories + assert len(repos) == 1 + [repo] = repos + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://security.ubuntu.com/ubuntu" + assert repo.release == "noble-security" + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + + def test_parse_deb822_paragraph_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + ok_para, bad_para = apt._iter_deb822_stanzas(lines) + repos = apt._Deb822Stanza(ok_para).repositories + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "" + with self.assertRaises(apt.InvalidSourceError): + apt._Deb822Stanza(bad_para) + + def test_parse_deb822_lines_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + print(repos[0].__dict__) + assert len(repos) == 4 + assert not errors + + def test_parse_deb822_lines_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + assert len(repos) == 3 + assert len(errors) == 1 + [error] = errors + assert isinstance(error, apt.InvalidSourceError) + + def test_load_deb822_ubuntu_sources(self): + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): + repository_mapping.load_deb822("") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + "deb-http://security.ubuntu.com/ubuntu-noble-security", + ] + + def test_load_deb822_w_comments(self): + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + + with patch( + "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments + ): + with patch.object(apt.logger, "debug") as debug: + repository_mapping.load_deb822("FILENAME") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + ] + debug.assert_called_once_with( + ANY, + 1, # number of errors + "Missing key 'Types' for entry starting on line 11 in FILENAME.", + ) + + def test_init_with_deb822(self): + """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. + + They should be equivalent with the sample data being used. + """ + + def isnt_file(f: str) -> bool: + return False + + def iglob_list(s: str) -> typing.Iterable[str]: + if s.endswith(".list"): + return ["FILENAME"] + return [] + + def iglob_sources(s: str) -> typing.Iterable[str]: + if s.endswith(".sources"): + return ["FILENAME"] + return [] + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_list): + repository_mapping_list = apt.RepositoryMapping() + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_sources): + repository_mapping_sources = apt.RepositoryMapping() + + list_keys = sorted(repository_mapping_list._repository_map.keys()) + sources_keys = sorted(repository_mapping_sources._repository_map.keys()) + assert sources_keys == list_keys + + for list_key, sources_key in zip(list_keys, sources_keys): + list_repo = repository_mapping_list[list_key] + sources_repo = repository_mapping_sources[sources_key] + assert list_repo.enabled == sources_repo.enabled + assert list_repo.repotype == sources_repo.repotype + assert list_repo.uri == sources_repo.uri + assert list_repo.release == sources_repo.release + assert list_repo.groups == sources_repo.groups + assert list_repo.gpg_key == sources_repo.gpg_key + assert ( + list_repo.options # pyright: ignore[reportUnknownMemberType] + == sources_repo.options # pyright: ignore[reportUnknownMemberType] + ) From c963280e05e041c44fbc9ec43f9c5d7f9449c579 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 27 Nov 2024 17:41:20 +1300 Subject: [PATCH 19/50] fix: remove extraneous information from docstring --- lib/charms/operator_libs_linux/v0/apt.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index bb55c758..97d6fabc 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1059,17 +1059,6 @@ def disable(self) -> None: WARNING: disable is currently not implemented for repositories defined by a deb822 stanza. Raises a NotImplementedError in this case. - - Potential semantics for disable on a deb822 defined repository are - described below. Please open an issue if you require the ability to - disable deb822 defined repositories. - - Multiple DebianRepository objects may be defined by the same stanza. - If the stanza that defines this object defines other DebianRepository - objects, an error will be raised on attempting to disable this repository, - unless an `always_disable_deb822_stanza` is passed, in which case the entire - stanza should be disabled -- which will also disable all other repositories - defined by the same stanza. """ if self._deb822_stanza is not None: raise NotImplementedError( From 7fa6295cc67ee29dd1a61de5588d48fe10ad8467 Mon Sep 17 00:00:00 2001 From: James Garner Date: Fri, 29 Nov 2024 11:54:37 +1300 Subject: [PATCH 20/50] feat: use apt-add-repository for RepositoryMapping.add --- lib/charms/operator_libs_linux/v0/apt.py | 115 +++++++++++------------ tests/unit/test_repo.py | 3 + 2 files changed, 56 insertions(+), 62 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 97d6fabc..aecc0991 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -108,9 +108,10 @@ import re import subprocess import textwrap +import typing from enum import Enum from subprocess import PIPE, CalledProcessError, check_output -from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Iterator, List, Literal, Mapping, Optional, Tuple, Union from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -1408,72 +1409,62 @@ def _parse(line: str, filename: str) -> DebianRepository: else: raise InvalidSourceError("An invalid sources line was found in %s!", filename) - def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: - """Add a new repository to the system. May have destructive or surprising behaviour. + @typing.overload + def add( # pyright: ignore[reportOverlappingOverload] + self, + repo: DebianRepository, + default_filename: Optional[bool] = False, + update_cache: Literal[False] = False, + ) -> None: ... + @typing.overload + def add( + self, + repo: DebianRepository, + default_filename: Optional[bool] = False, + update_cache: Literal[True] = True, + ) -> "RepositoryMapping": ... + def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented + self, + repo: DebianRepository, + default_filename: Optional[bool] = False, + update_cache: bool = False, + ) -> Optional["RepositoryMapping"]: + """Add a new repository to the system using apt-add-repository. + + Args: + repo: a DebianRepository object where repo.enabled is True + update_cache: if True, apt-add-repository will update the package cache + and then a new RepositoryMapping object will be returned. + If False, then apt-add-repository is run with the --no-update option, + and an entry for repo is added to this RepositoryMapping + Returns: + None, or a new RepositoryMapping object if update_cache is True - Will write in the deb822 format if: - 1. repo.filename has the '.sources' extension - 2. repo.filename is unset (""), but /etc/apt/sources.list.d/ubuntu.sources exists - Otherwise the one-per-line style will be used. + raises: + ValueError if repo.enabled is False + CalledProcessError if there's an error running apt-add-repository WARNING: the default_filename keyword argument is provided for backwards compatibility only. It is not used, and was not used in the previous revision of this library. - - WARNING: in the one-per-line format case, will mutate repo.options to add the 'signed-by' - key if both options and gpg_file are truthy (for example if a gpg_key_filename and - non-empty options were provided at DebianRepository initialisation time). If options were - not provided or are empty, but a gpg_key is available, will silently fail to include it. - - WARNING: if repo.filename is falsey, the new filename is calculated only from the repo's - uri and release, but repos are assumed to be uniquely identified by the combination of - repotype, uri, and release. Adding two repos with differing repotypes but identical uris - and releases will result in the second repo clobbering the file written by the first. - In this case, repo.filename must be set appropriately to avoid data loss. - - WARNING: if repo.filename is truthy, and that file exists, this method will clobber that - file with a single entry for the repo being added. Set it to a falsey value to have a new - filename derived from its uri and release (which will also clobber any existing file), or - set the filename that you want to write to -- or construct a new DebianRepository object - with the filename you want to write to. - For example: repo.filename = my_filename - For example: DebianRepository(uri=repo.uri, filename=my_filename, ...) - - WARNING: if repo.enabled is falsey, the repository will be added in the disabled state. - Note that repo.disable() does not affect this value. If repo.enabled does not match the - value you expect, construct a new DebianRepository object with the appropriate value. - For example: DebianRepository(uri=repo.uri, enabled=my_value, ...) """ - if repo.filename: - fname = repo.filename - else: - fname = "{}-{}.{}".format( - DebianRepository.prefix_from_uri(repo.uri), - repo.release.replace("/", "-"), - "sources" if os.path.isfile(self.default_sources) else "list", - ) - - _, fname_extension = fname.rsplit(".", maxsplit=1) - if fname_extension == "sources": - text = "\n".join( - "{}: {}".format(key, value) - for key, value in _opts_from_repository( - repo, - stanza=repo._deb822_stanza, # pyright: ignore[reportPrivateUsage] - ) - ) - else: - options = repo.options if repo.options else {} - if repo.gpg_key: - options["signed-by"] = repo.gpg_key - text = ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, repo.make_options_string(), repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) - ) - - with open(fname, "wb") as f: - f.write(text.encode("utf-8")) - + if not repo.enabled: + raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) + cmd = [ + "apt-add-repository", + "--dry-run", + "--yes", + f"--uri={repo.uri}", + f"--pocket={repo.release}", + ] + if repo.repotype == "deb-src": + cmd.append("--enable-source") + for component in repo.groups: + cmd.append(f"--component={component}") + if not update_cache: + cmd.append("--no-update") + subprocess.run(cmd, check=True) + if update_cache: + return RepositoryMapping() self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo def disable(self, repo: DebianRepository) -> None: diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index cc17d680..9e0cf537 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -8,6 +8,7 @@ import unittest from unittest.mock import ANY, mock_open, patch +import pytest from charms.operator_libs_linux.v0 import apt from pyfakefs.fake_filesystem_unittest import TestCase as PyFakeFsTestCase @@ -93,6 +94,7 @@ def test_can_disable_repositories(self): open(other.filename).readlines(), ) + @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") def test_can_add_repositories(self): r = apt.RepositoryMapping() d = apt.DebianRepository( @@ -119,6 +121,7 @@ def test_can_add_repositories_from_string(self): self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") self.assertIn("deb https://example.com/foo focal bar baz\n", open(d.filename).readlines()) + @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") def test_valid_list_file(self): line = "deb https://repo.example.org/fiz/baz focal/foo-bar/5.0 multiverse" d = apt.DebianRepository.from_repo_line(line) From 86ebdd1ca4f54c7923d1a605d6aebd213b1ed06c Mon Sep 17 00:00:00 2001 From: James Garner Date: Mon, 2 Dec 2024 11:20:05 +1300 Subject: [PATCH 21/50] refactor: logic and testing cleanup --- lib/charms/operator_libs_linux/v0/apt.py | 370 +++++++++++------------ tests/unit/test_deb822.py | 299 ++++++++++++++++++ tests/unit/test_repo.py | 298 +----------------- 3 files changed, 474 insertions(+), 493 deletions(-) create mode 100644 tests/unit/test_deb822.py diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index aecc0991..b33b46cf 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -107,7 +107,6 @@ import os import re import subprocess -import textwrap import typing from enum import Enum from subprocess import PIPE, CalledProcessError, check_output @@ -915,6 +914,9 @@ class GPGKeyError(Error): class DebianRepository: """An abstraction to represent a repository.""" + _deb822_stanza: Optional["_Deb822Stanza"] = None + """set by Deb822Stanza after creating a DebianRepository""" + def __init__( self, enabled: bool, @@ -925,7 +927,6 @@ def __init__( filename: Optional[str] = "", gpg_key_filename: Optional[str] = "", options: Optional[Dict[str, str]] = None, - deb822_stanza: Optional["_Deb822Stanza"] = None, ): self._enabled = enabled self._repotype = repotype @@ -935,7 +936,6 @@ def __init__( self._filename = filename self._gpg_key_filename = gpg_key_filename self._options = options - self._deb822_stanza = deb822_stanza @property def enabled(self): @@ -997,14 +997,15 @@ def make_options_string(self) -> str: a complex repo string. """ options = self._options if self._options else {} - if self._gpg_key_filename: - options["signed-by"] = self._gpg_key_filename - return ( - "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in options.items()])) - if options - else "" - ) + gpg_key_filename = self.gpg_key # ensure getter logic is run + if gpg_key_filename: + options["signed-by"] = gpg_key_filename + + if not options: + return "" + + return "[{}] ".format(" ".join("{}={}".format(k, v) for k, v in options.items())) @staticmethod def prefix_from_uri(uri: str) -> str: @@ -1063,20 +1064,8 @@ def disable(self) -> None: """ if self._deb822_stanza is not None: raise NotImplementedError( - textwrap.dedent( - """ - Disabling a repository defined by a deb822 format source is not implemented. - Please raise an issue if you require this feature. - This repository is: - {str_self} - Defined by the deb822 stanza: - {str_stanza} - ) - """.format( - str_self=textwrap.indent(str(self), " " * 2), - str_stanza=textwrap.indent(str(self._deb822_stanza), " " * 2), - ) - ).strip() + "Disabling a repository defined by a deb822 format source is not implemented." + " Please raise an issue if you require this feature." ) searcher = "{} {}{} {}".format( self.repotype, self.make_options_string(), self.uri, self.release @@ -1334,7 +1323,7 @@ def load(self, filename: str): parsed: List[int] = [] skipped: List[int] = [] with open(filename, "r") as f: - for n, line in enumerate(f): + for n, line in enumerate(f, start=1): # 1 indexed line numbers try: repo = self._parse(line, filename) except InvalidSourceError: @@ -1436,13 +1425,18 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und update_cache: if True, apt-add-repository will update the package cache and then a new RepositoryMapping object will be returned. If False, then apt-add-repository is run with the --no-update option, - and an entry for repo is added to this RepositoryMapping + and an entry for repo is added to this RepositoryMapping, and you + can call `update` manually before installing any packages. + Returns: None, or a new RepositoryMapping object if update_cache is True raises: - ValueError if repo.enabled is False - CalledProcessError if there's an error running apt-add-repository + ValueError: if repo.enabled is False + CalledProcessError: if there's an error running apt-add-repository + + WARNING: Does not associate the repository with a signing key. + Use the `import_key` method to add a signing key globally. WARNING: the default_filename keyword argument is provided for backwards compatibility only. It is not used, and was not used in the previous revision of this library. @@ -1487,16 +1481,26 @@ class _Deb822Stanza: def __init__(self, numbered_lines: List[Tuple[int, str]], filename: str = ""): self._numbered_lines = numbered_lines self._filename = filename - self._gpg_key_from_stanza: Optional[str] = None # may be set in _parse_stanzas - self._gpg_key_filename: Optional[str] = None # may be set in _parse_stanzas - self._repositories = self._parse_stanza(lines=numbered_lines, filename=filename) + self._options, self._line_numbers = _deb822_stanza_to_options(numbered_lines) + repos, gpg_key_info = _deb822_options_to_repos( + self._options, line_numbers=self._line_numbers, filename=filename + ) + for repo in repos: + repo._deb822_stanza = self # pyright: ignore[reportPrivateUsage] + self._repositories = repos + self._gpg_key_filename, self._gpg_key_from_stanza = gpg_key_info + + @property + def repositories(self) -> Tuple[DebianRepository, ...]: + """The repositories defined by this deb822 stanza.""" + return self._repositories def get_gpg_key_filename(self) -> str: """Return the path to the GPG key for this repository. Import the key first, if the key itself was provided in the stanza. """ - if self._gpg_key_filename is not None: + if self._gpg_key_filename: return self._gpg_key_filename if self._gpg_key_from_stanza is None: return "" @@ -1505,152 +1509,19 @@ def get_gpg_key_filename(self) -> str: self._gpg_key_filename = import_key(self._gpg_key_from_stanza) return self._gpg_key_filename - @property - def repositories(self) -> Tuple[DebianRepository, ...]: - """The repositories defined by this deb822 stanza.""" - return self._repositories - - def __str__(self) -> str: - """Return formatted representation of object instantiation.""" - return textwrap.dedent( - """ - {name}( - filename={filename}, - numbered_lines=[ - {lines} - ], - ) - """.format( - name=self.__class__.__name__, - filename=self._filename, - lines="\n ".join(str(line) for line in self._numbered_lines), - ) - ).strip() - - def _parse_stanza( - self, lines: List[Tuple[int, str]], filename: str = "" - ) -> Tuple[DebianRepository, ...]: - """Return a list of DebianRepository objects defined by this deb822 stanza. - - Raises: - InvalidSourceError if the source type is unknown or contains malformed entries - """ - options, line_numbers = self._get_options(lines) - # Enabled - enabled_field = options.pop("Enabled", "yes") - if enabled_field == "yes": - enabled = True - elif enabled_field == "no": - enabled = False - else: - raise InvalidSourceError( - ( - "Bad value '{value}' for entry 'Enabled' (line {enabled_line})" - " in file {file}. If 'Enabled' is present it must be one of" - " yes or no (if absent it defaults to yes)." - ).format( - value=enabled_field, - enabled_line=line_numbers["Enabled"], - file=filename, - ) - ) - # Signed-By - gpg_key_file = options.pop("Signed-By", "") - if "\n" in gpg_key_file: - # actually a literal multi-line gpg-key rather than a filename - self._gpg_key_from_stanza = gpg_key_file - gpg_key_file = "" - # Types - try: - repotypes = options.pop("Types").split() - uris = options.pop("URIs").split() - suites = options.pop("Suites").split() - except KeyError as e: - [key] = e.args - raise InvalidSourceError( - "Missing key '{key}' for entry starting on line {line} in {file}.".format( - key=key, - line=min(line_numbers.values()), - file=filename, - ) - ) - # Components - components: List[str] - if len(suites) == 1 and suites[0].endswith("/"): - if "Components" in options: - raise InvalidSourceError( - ( - "Since 'Suites' (line {suites_line}) specifies" - " a path relative to 'URIs' (line {uris_line})," - " 'Components' (line {components_line}) must be ommitted" - " (in file {file})." - ).format( - suites_line=line_numbers["Suites"], - uris_line=line_numbers["URIs"], - components_line=line_numbers["Components"], - file=filename, - ) - ) - components = [] - else: - if "Components" not in options: - raise InvalidSourceError( - ( - "Since 'Suites' (line {suites_line}) does not specify" - " a path relative to 'URIs' (line {uris_line})," - " 'Components' must be present in this paragraph" - " (in file {file})." - ).format( - suites_line=line_numbers["Suites"], - uris_line=line_numbers["URIs"], - file=filename, - ) - ) - components = options.pop("Components").split() - return tuple( - DebianRepository( - enabled=enabled, - repotype=repotype, - uri=uri, - release=suite, - groups=components, - filename=filename, - gpg_key_filename=gpg_key_file, - options=options, - deb822_stanza=self, - ) - for repotype, uri, suite in itertools.product(repotypes, uris, suites) - ) - - @staticmethod - def _get_options( - lines: Iterable[Tuple[int, str]], - ) -> Tuple[Dict[str, str], Dict[str, int]]: - parts: Dict[str, List[str]] = {} - line_numbers: Dict[str, int] = {} - current = None - for n, line in lines: - assert "#" not in line # comments should be stripped out - if line.startswith(" "): # continuation of previous key's value - assert current is not None - parts[current].append(line.rstrip()) # preserve indent - continue - raw_key, _, raw_value = line.partition(":") - current = raw_key.strip() - parts[current] = [raw_value.strip()] - line_numbers[current] = n - options = {k: "\n".join(v) for k, v in parts.items()} - return options, line_numbers - def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: """Given lines from a deb822 format file, yield a stanza of lines. - A stanza is a list of numbered lines that make up a source entry, - with comments stripped out (but accounted for in line numbering). + Args: + lines: an iterable of lines from a deb822 sources file + + Yields: + lists of numbered lines (a tuple of line number and line) that make up + a deb822 stanza, with comments stripped out (but accounted for in line numbering) """ current_paragraph: List[Tuple[int, str]] = [] - for n, line in enumerate(lines): # 0 indexed line numbers, following `load` + for n, line in enumerate(lines, start=1): # 1 indexed line numbers if not line.strip(): # blank lines separate paragraphs if current_paragraph: yield current_paragraph @@ -1663,30 +1534,135 @@ def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]] yield current_paragraph -def _opts_from_repository( - repo: DebianRepository, stanza: Optional[_Deb822Stanza] = None -) -> Dict[str, str]: - """Return repo information as deb822 format keys and values.""" - opts = { - "Enabled": ("yes" if repo.enabled else "no"), - "Types": repo.repotype, - "URIs": repo.uri, - "Suites": repo.release, - } +def _deb822_stanza_to_options( + lines: Iterable[Tuple[int, str]], +) -> Tuple[Dict[str, str], Dict[str, int]]: + """Turn numbered lines into a dict of options and a dict of line numbers. + + Args: + lines: an iterable of numbered lines (a tuple of line number and line) + + Returns: + a dictionary of option names to (potentially multiline) values, and + a dictionary of option names to starting line number + """ + parts: Dict[str, List[str]] = {} + line_numbers: Dict[str, int] = {} + current = None + for n, line in lines: + assert "#" not in line # comments should be stripped out + if line.startswith(" "): # continuation of previous key's value + assert current is not None + parts[current].append(line.rstrip()) # preserve indent + continue + raw_key, _, raw_value = line.partition(":") + current = raw_key.strip() + parts[current] = [raw_value.strip()] + line_numbers[current] = n + options = {k: "\n".join(v) for k, v in parts.items()} + return options, line_numbers + + +def _deb822_options_to_repos( + options: Dict[str, str], line_numbers: Mapping[str, int] = {}, filename: str = "" +) -> Tuple[Tuple[DebianRepository, ...], Tuple[str, Optional[str]]]: + """Return a collections of DebianRepository objects defined by this deb822 stanza. + + Args: + options: a dictionary of deb822 field names to string options + line_numbers: a dictionary of field names to line numbers (for error messages) + filename: the file the options were read from (for repository object and errors) + + Returns: + a tuple of `DebianRepository`s, and + a tuple of the gpg key filename and optional in-stanza provided key itself + + Raises: + InvalidSourceError if any options are malformed or required options are missing + """ + # Enabled + enabled_field = options.pop("Enabled", "yes") + if enabled_field == "yes": + enabled = True + elif enabled_field == "no": + enabled = False + else: + raise InvalidSourceError( + ( + "Bad value '{value}' for entry 'Enabled' (line {enabled_line})" + " in file {file}. If 'Enabled' is present it must be one of" + " yes or no (if absent it defaults to yes)." + ).format( + value=enabled_field, + enabled_line=line_numbers.get("Enabled"), + file=filename, + ) + ) # Signed-By - # use the actual gpg key if it was provided in the stanza - # otherwise use the filename if that was provided - gpg_key: Optional[str] = None - if stanza is not None: - gpg_key = stanza._gpg_key_from_stanza - if gpg_key is None: # no inline gpg key provided - gpg_key = repo.gpg_key - if gpg_key: - opts["Signed-By"] = gpg_key + gpg_key_file = options.pop("Signed-By", "") + gpg_key_from_stanza: Optional[str] = None + if "\n" in gpg_key_file: + # actually a literal multi-line gpg-key rather than a filename + gpg_key_from_stanza = gpg_key_file + gpg_key_file = "" + # Types + try: + repotypes = options.pop("Types").split() + uris = options.pop("URIs").split() + suites = options.pop("Suites").split() + except KeyError as e: + [key] = e.args + raise InvalidSourceError( + "Missing required entry '{key}' for entry starting on line {line} in {file}.".format( + key=key, + line=min(line_numbers.values()) if line_numbers else None, + file=filename, + ) + ) # Components - if repo.groups: - opts["Components"] = " ".join(repo.groups) - # options - if repo.options is not None: - opts.update(repo.options) - return opts + components: List[str] + if len(suites) == 1 and suites[0].endswith("/"): + if "Components" in options: + raise InvalidSourceError( + ( + "Since 'Suites' (line {suites_line}) specifies" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' (line {components_line}) must be ommitted" + " (in file {file})." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + components_line=line_numbers.get("Components"), + file=filename, + ) + ) + components = [] + else: + if "Components" not in options: + raise InvalidSourceError( + ( + "Since 'Suites' (line {suites_line}) does not specify" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be present in this paragraph" + " (in file {file})." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + file=filename, + ) + ) + components = options.pop("Components").split() + repos = tuple( + DebianRepository( + enabled=enabled, + repotype=repotype, + uri=uri, + release=suite, + groups=components, + filename=filename, + gpg_key_filename=gpg_key_file, + options=options, + ) + for repotype, uri, suite in itertools.product(repotypes, uris, suites) + ) + return repos, (gpg_key_file, gpg_key_from_stanza) diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py new file mode 100644 index 00000000..8d093261 --- /dev/null +++ b/tests/unit/test_deb822.py @@ -0,0 +1,299 @@ +# Copyright 2021 Canonical Ltd. +# See LICENSE file for licensing details. + +# pyright: reportPrivateUsage=false + +import typing +import unittest +from unittest.mock import ANY, mock_open, patch + +from charms.operator_libs_linux.v0 import apt + +ubuntu_sources_deb822 = """ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +""" + +ubuntu_sources_deb822_with_comments = """ +Components: main restricted universe multiverse # this lib doesn't care about order +Types: deb # this could include deb-src as well or instead +URIs: http://nz.archive.ubuntu.com/ubuntu/ + # there can be multiple space separated URIs + # sources are specified in priority order + # apt does some de-duplication of sources after parsing too +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +# let's make this insecure! (jk, just testing parsing) +Suites: noble noble-updates noble-backports + +Foo: Bar # this is a separate (malformed) entry + +#Types: deb +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +## disable security updates while we're at it +""" + +ubuntu_sources_one_line = """ +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse +""" + +inkscape_sources_deb822 = """ +Types: deb +URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ +Suites: noble +Components: main +Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- +""" + + +class TestRepositoryMappingDeb822Behaviour(unittest.TestCase): + def test_iter_deb822_paragraphs_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + stanzas = list(apt._iter_deb822_stanzas(lines)) + assert len(stanzas) == 2 + stanza_1, stanza_2 = stanzas + assert len(stanza_1) == 5 + assert len(stanza_2) == 5 + line_numbers = [n for stanza in stanzas for n, _line in stanza] + assert len(set(line_numbers)) == len(line_numbers) # unique line numbers + + def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + stanzas = list(apt._iter_deb822_stanzas(lines)) + assert len(stanzas) == 2 + stanza_1, stanza_2 = stanzas + assert len(stanza_1) == 4 + assert len(stanza_2) == 1 + line_numbers = [n for stanza in stanzas for n, _line in stanza] + assert len(set(line_numbers)) == len(line_numbers) # unique line numbers + + def test_get_deb822_options_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + opts = [apt._deb822_stanza_to_options(p) for p in paras] + opts_0, opts_1 = opts + opts_0_options, _opts_0_line_numbers = opts_0 + opts_1_options, _opts_1_line_numbers = opts_1 + assert opts_0_options == { + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Components": "main restricted universe multiverse", + "Suites": "noble noble-updates noble-backports", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", + } + assert opts_1_options == { + "Types": "deb", + "URIs": "http://security.ubuntu.com/ubuntu", + "Components": "main restricted universe multiverse", + "Suites": "noble-security", + "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", + } + + def test_get_deb822_options_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + paras = list(apt._iter_deb822_stanzas(lines)) + opts = [apt._deb822_stanza_to_options(p) for p in paras] + opts_0, opts_1 = opts + opts_0_options, _opts_0_line_numbers = opts_0 + opts_1_options, _opts_1_line_numbers = opts_1 + assert opts_0_options == { + "Components": "main restricted universe multiverse", + "Types": "deb", + "URIs": "http://nz.archive.ubuntu.com/ubuntu/", + "Suites": "noble noble-updates noble-backports", + } + assert opts_1_options == {"Foo": "Bar"} + + def test_parse_deb822_paragraph_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + main, security = apt._iter_deb822_stanzas(lines) + repos = apt._Deb822Stanza(main).repositories + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + repos = apt._Deb822Stanza(security).repositories + assert len(repos) == 1 + [repo] = repos + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://security.ubuntu.com/ubuntu" + assert repo.release == "noble-security" + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + + def test_parse_deb822_paragraph_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + ok_stanza, bad_stanza = apt._iter_deb822_stanzas(lines) + repos = apt._Deb822Stanza(ok_stanza).repositories + assert len(repos) == 3 + for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == "" + assert repo.gpg_key == "" + with self.assertRaises(apt.InvalidSourceError): + apt._Deb822Stanza(bad_stanza) + + def test_parse_deb822_lines_ubuntu_sources(self): + lines = ubuntu_sources_deb822.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + assert len(repos) == 4 + assert not errors + + def test_parse_deb822_lines_w_comments(self): + lines = ubuntu_sources_deb822_with_comments.strip().split("\n") + repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) + assert len(repos) == 3 + assert len(errors) == 1 + [error] = errors + assert isinstance(error, apt.InvalidSourceError) + + def test_load_deb822_ubuntu_sources(self): + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): + repository_mapping.load_deb822("") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + "deb-http://security.ubuntu.com/ubuntu-noble-security", + ] + + def test_load_deb822_w_comments(self): + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + + with patch( + "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments + ): + with patch.object(apt.logger, "debug") as debug: + repository_mapping.load_deb822("FILENAME") + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + ] + debug.assert_called_once_with( + ANY, + 1, # number of errors + "Missing required entry 'Types' for entry starting on line 12 in FILENAME.", + ) + + def test_init_with_deb822(self): + """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. + + They should be equivalent with the sample data being used. + """ + + def isnt_file(f: str) -> bool: + return False + + def iglob_list(s: str) -> typing.Iterable[str]: + if s.endswith(".list"): + return ["FILENAME"] + return [] + + def iglob_sources(s: str) -> typing.Iterable[str]: + if s.endswith(".sources"): + return ["FILENAME"] + return [] + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_list): + repository_mapping_list = apt.RepositoryMapping() + + with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_sources): + repository_mapping_sources = apt.RepositoryMapping() + + list_keys = sorted(repository_mapping_list._repository_map.keys()) + sources_keys = sorted(repository_mapping_sources._repository_map.keys()) + assert sources_keys == list_keys + + for list_key, sources_key in zip(list_keys, sources_keys): + list_repo = repository_mapping_list[list_key] + sources_repo = repository_mapping_sources[sources_key] + assert list_repo.enabled == sources_repo.enabled + assert list_repo.repotype == sources_repo.repotype + assert list_repo.uri == sources_repo.uri + assert list_repo.release == sources_repo.release + assert list_repo.groups == sources_repo.groups + assert list_repo.gpg_key == sources_repo.gpg_key + assert ( + list_repo.options # pyright: ignore[reportUnknownMemberType] + == sources_repo.options # pyright: ignore[reportUnknownMemberType] + ) + diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index 9e0cf537..29061262 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -1,16 +1,11 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. -# pyright: reportPrivateUsage=false - import os -import typing -import unittest -from unittest.mock import ANY, mock_open, patch import pytest from charms.operator_libs_linux.v0 import apt -from pyfakefs.fake_filesystem_unittest import TestCase as PyFakeFsTestCase +from pyfakefs.fake_filesystem_unittest import TestCase sources_list = """## This is a comment which should be ignored! deb http://us.archive.ubuntu.com/ubuntu focal main restricted universe multiverse @@ -36,7 +31,7 @@ """ -class TestRepositoryMapping(PyFakeFsTestCase): +class TestRepositoryMapping(TestCase): def setUp(self): self.setUpPyfakefs() self.fs.create_file("/etc/apt/sources.list", contents=sources_list) @@ -154,292 +149,3 @@ def test_can_add_repositories_from_string_with_options(self): "deb [arch=amd64 signed-by=/foo/gpg.key] https://example.com/foo focal bar baz\n", open(d.filename).readlines(), ) - - -ubuntu_sources_deb822 = """ -Types: deb -URIs: http://nz.archive.ubuntu.com/ubuntu/ -Suites: noble noble-updates noble-backports -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg - -Types: deb -URIs: http://security.ubuntu.com/ubuntu -Suites: noble-security -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -""" - -ubuntu_sources_deb822_with_comments = """ -Components: main restricted universe multiverse # this lib doesn't care about order -Types: deb # this could include deb-src as well or instead -URIs: http://nz.archive.ubuntu.com/ubuntu/ - # there can be multiple space separated URIs - # sources are specified in priority order - # apt does some de-duplication of sources after parsing too -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -# let's make this insecure! (jk, just testing parsing) -Suites: noble noble-updates noble-backports - -Foo: Bar # this is a separate (malformed) entry - -#Types: deb -#URIs: http://security.ubuntu.com/ubuntu -#Suites: noble-security -#Components: main restricted universe multiverse -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -## disable security updates while we're at it -""" - -ubuntu_sources_one_line = """ -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse -""" - - -class TestRepositoryMappingDeb822Behaviour(unittest.TestCase): - def test_iter_deb822_paragraphs_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - assert paras == [ - [ - (0, "Types: deb"), - (1, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), - (2, "Suites: noble noble-updates noble-backports"), - (3, "Components: main restricted universe multiverse"), - (4, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), - ], - [ - (6, "Types: deb"), - (7, "URIs: http://security.ubuntu.com/ubuntu"), - (8, "Suites: noble-security"), - (9, "Components: main restricted universe multiverse"), - (10, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"), - ], - ] - - def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - assert paras == [ - [ - (0, "Components: main restricted universe multiverse"), - (1, "Types: deb"), - (2, "URIs: http://nz.archive.ubuntu.com/ubuntu/"), - (8, "Suites: noble noble-updates noble-backports"), - ], - [ - (10, "Foo: Bar"), - ], - ] - - def test_get_deb822_options_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - opts = [apt._Deb822Stanza._get_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, opts_0_line_numbers = opts_0 - opts_1_options, opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Components": "main restricted universe multiverse", - "Suites": "noble noble-updates noble-backports", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - assert opts_0_line_numbers == { - "Types": 0, - "URIs": 1, - "Suites": 2, - "Components": 3, - "Signed-By": 4, - } - assert opts_1_options == { - "Types": "deb", - "URIs": "http://security.ubuntu.com/ubuntu", - "Components": "main restricted universe multiverse", - "Suites": "noble-security", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - assert opts_1_line_numbers == { - "Types": 6, - "URIs": 7, - "Suites": 8, - "Components": 9, - "Signed-By": 10, - } - - def test_get_deb822_options_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - opts = [apt._Deb822Stanza._get_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, opts_0_line_numbers = opts_0 - opts_1_options, opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Components": "main restricted universe multiverse", - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Suites": "noble noble-updates noble-backports", - } - assert opts_0_line_numbers == { - "Components": 0, - "Types": 1, - "URIs": 2, - "Suites": 8, - } - assert opts_1_options == {"Foo": "Bar"} - assert opts_1_line_numbers == {"Foo": 10} - - def test_parse_deb822_paragraph_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - main, security = apt._iter_deb822_stanzas(lines) - repos = apt._Deb822Stanza(main).repositories - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - repos = apt._Deb822Stanza(security).repositories - assert len(repos) == 1 - [repo] = repos - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://security.ubuntu.com/ubuntu" - assert repo.release == "noble-security" - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - - def test_parse_deb822_paragraph_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - ok_para, bad_para = apt._iter_deb822_stanzas(lines) - repos = apt._Deb822Stanza(ok_para).repositories - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "" - with self.assertRaises(apt.InvalidSourceError): - apt._Deb822Stanza(bad_para) - - def test_parse_deb822_lines_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - print(repos[0].__dict__) - assert len(repos) == 4 - assert not errors - - def test_parse_deb822_lines_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - assert len(repos) == 3 - assert len(errors) == 1 - [error] = errors - assert isinstance(error, apt.InvalidSourceError) - - def test_load_deb822_ubuntu_sources(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - repository_mapping.load_deb822("") - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - "deb-http://security.ubuntu.com/ubuntu-noble-security", - ] - - def test_load_deb822_w_comments(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch( - "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments - ): - with patch.object(apt.logger, "debug") as debug: - repository_mapping.load_deb822("FILENAME") - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - ] - debug.assert_called_once_with( - ANY, - 1, # number of errors - "Missing key 'Types' for entry starting on line 11 in FILENAME.", - ) - - def test_init_with_deb822(self): - """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. - - They should be equivalent with the sample data being used. - """ - - def isnt_file(f: str) -> bool: - return False - - def iglob_list(s: str) -> typing.Iterable[str]: - if s.endswith(".list"): - return ["FILENAME"] - return [] - - def iglob_sources(s: str) -> typing.Iterable[str]: - if s.endswith(".sources"): - return ["FILENAME"] - return [] - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_list): - repository_mapping_list = apt.RepositoryMapping() - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_sources): - repository_mapping_sources = apt.RepositoryMapping() - - list_keys = sorted(repository_mapping_list._repository_map.keys()) - sources_keys = sorted(repository_mapping_sources._repository_map.keys()) - assert sources_keys == list_keys - - for list_key, sources_key in zip(list_keys, sources_keys): - list_repo = repository_mapping_list[list_key] - sources_repo = repository_mapping_sources[sources_key] - assert list_repo.enabled == sources_repo.enabled - assert list_repo.repotype == sources_repo.repotype - assert list_repo.uri == sources_repo.uri - assert list_repo.release == sources_repo.release - assert list_repo.groups == sources_repo.groups - assert list_repo.gpg_key == sources_repo.gpg_key - assert ( - list_repo.options # pyright: ignore[reportUnknownMemberType] - == sources_repo.options # pyright: ignore[reportUnknownMemberType] - ) From c3641110b93b03f62aee8c8fee3a97ee3fc8463d Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 12:57:56 +1300 Subject: [PATCH 22/50] style: type annotate integration test helper function --- tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index d481a56b..8f89d133 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -5,7 +5,7 @@ from subprocess import CalledProcessError, check_output -def get_command_path(command): +def get_command_path(command: str) -> str: try: return check_output(["which", command]).decode().strip() except CalledProcessError: From 71c5f905c74cf0ade937dceb3505d6093f63c12b Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 12:59:14 +1300 Subject: [PATCH 23/50] feat: support apt autoremove in remove_package --- lib/charms/operator_libs_linux/v0/apt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index b33b46cf..d861b85c 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -807,12 +807,17 @@ def _add( def remove_package( - package_names: Union[str, List[str]] + package_names: Union[str, List[str]], + autoremove: bool = False, ) -> Union[DebianPackage, List[DebianPackage]]: """Remove package(s) from the system. Args: package_names: the name of a package + autoremove: run `apt autoremove` after uninstalling packages. + You probably want to do this if you're removing a metapackage, + otherwise the concrete packages will still be there. + False by default for backwards compatibility. Raises: PackageNotFoundError if the package is not found. @@ -831,6 +836,9 @@ def remove_package( except PackageNotFoundError: logger.info("package '%s' was requested for removal, but it was not installed.", p) + if autoremove: + subprocess.run(["apt", "autoremove"], check=True) + # the list of packages will be empty when no package is removed logger.debug("packages: '%s'", packages) return packages[0] if len(packages) == 1 else packages From afc85b6a5803bb68733ae50adbc0948c4a1d3daf Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 12:59:51 +1300 Subject: [PATCH 24/50] feat: debug useful information on error in apt.update --- lib/charms/operator_libs_linux/v0/apt.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index d861b85c..47a8f857 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -846,7 +846,17 @@ def remove_package( def update() -> None: """Update the apt cache via `apt-get update`.""" - subprocess.run(["apt-get", "update", "--error-on=any"], capture_output=True, check=True) + cmd = ["apt-get", "update", "--error-on=any"] + try: + subprocess.run(cmd, capture_output=True, check=True) + except CalledProcessError as e: + logger.error( + "%s:\nstdout:\n%s\nstderr:\n%s", + " ".join(cmd), + e.stdout.decode(), + e.stderr.decode(), + ) + raise def import_key(key: str) -> str: From 3b7872a0658a9e6ca2530d8dc12f1fc89f51451c Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:01:33 +1300 Subject: [PATCH 25/50] feat: log when writing to a sources file in from_repo_line --- lib/charms/operator_libs_linux/v0/apt.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 47a8f857..6300b1c5 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1060,15 +1060,21 @@ def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "Debian else "" ) + repo_str = ( + "{}".format("#" if not repo.enabled else "") + + "{} {}{} ".format(repo.repotype, options_str, repo.uri) + + "{} {}\n".format(repo.release, " ".join(repo.groups)) + ) + if write_file: + logger.info( + "DebianRepository.from_repo_line('%s', write_file=True)\nWriting to '%s':\n%s", + repo_line, + fname, + repo_str, + ) with open(fname, "wb") as f: - f.write( - ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, options_str, repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) - ).encode("utf-8") - ) + f.write(repo_str.encode("utf-8")) return repo From e24001ae7617f371f989f06c43bc4b7b91ef97e7 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:02:30 +1300 Subject: [PATCH 26/50] feat: make the apt directories (private) attributes (for testing) --- lib/charms/operator_libs_linux/v0/apt.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 6300b1c5..b365fa05 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1235,10 +1235,17 @@ class RepositoryMapping(Mapping[str, DebianRepository]): )) """ + _apt_dir = "/etc/apt" + _sources_subdir = "sources.list.d" + _default_list_name = "sources.list" + _default_sources_name = "ubuntu.sources" + def __init__(self): self._repository_map: Dict[str, DebianRepository] = {} - self.default_file = "/etc/apt/sources.list" - self.default_sources = "/etc/apt/sources.list.d/ubuntu.sources" + self.default_file = os.path.join(self._apt_dir, self._default_list_name) + # ^ public attribute for backwards compatibility only + sources_dir = os.path.join(self._apt_dir, self._sources_subdir) + default_sources = os.path.join(sources_dir, self._default_sources_name) # read sources.list if it exists # ignore InvalidSourceError if ubuntu.sources also exists @@ -1247,13 +1254,13 @@ def __init__(self): try: self.load(self.default_file) except InvalidSourceError: - if not os.path.isfile(self.default_sources): + if not os.path.isfile(default_sources): raise # read sources.list.d - for file in glob.iglob("/etc/apt/sources.list.d/*.list"): + for file in glob.iglob(os.path.join(sources_dir, "*.list")): self.load(file) - for file in glob.iglob("/etc/apt/sources.list.d/*.sources"): + for file in glob.iglob(os.path.join(sources_dir, "*.sources")): self.load_deb822(file) def __contains__(self, key: Any) -> bool: From 497bf0e811ce9d01606818bfeffeeebfe6a48a43 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:03:32 +1300 Subject: [PATCH 27/50] style: correct RepositoryMapping.__iter__ annotation and add FIXME --- lib/charms/operator_libs_linux/v0/apt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index b365fa05..37ea4be1 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1274,10 +1274,12 @@ def __len__(self) -> int: """Return number of repositories in map.""" return len(self._repository_map) - def __iter__(self) -> Iterable[DebianRepository]: + def __iter__(self) -> Iterator[DebianRepository]: """Return iterator for RepositoryMapping. Iterates over the DebianRepository values rather than the string names. + FIXME: this breaks the expectations of the Mapping abstract base class + for example when it provides methods like keys and items """ return iter(self._repository_map.values()) From 7c70fa8698273aca206f3feed77340d662dc8e0e Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:04:34 +1300 Subject: [PATCH 28/50] feat: log the filename as well as the number of repos when parsing --- lib/charms/operator_libs_linux/v0/apt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 37ea4be1..21d0dce4 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1319,7 +1319,7 @@ def load_deb822(self, filename: str) -> None: ) if repos: - logger.info("parsed %d apt package repositories", len(repos)) + logger.info("parsed %d apt package repositories from %s", len(repos), filename) else: raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) @@ -1372,7 +1372,7 @@ def load(self, filename: str): logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) if parsed: - logger.info("parsed %d apt package repositories", len(parsed)) + logger.info("parsed %d apt package repositories from %s", len(parsed), filename) else: raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) From 10df1e8ce9271a518d007686db434bda67155e75 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:06:27 +1300 Subject: [PATCH 29/50] feat: refactor RepositoryMapping.add to possible support remove as well --- lib/charms/operator_libs_linux/v0/apt.py | 81 ++++++++++++++++-------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 21d0dce4..a2e258c0 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -107,10 +107,9 @@ import os import re import subprocess -import typing from enum import Enum from subprocess import PIPE, CalledProcessError, check_output -from typing import Any, Dict, Iterable, Iterator, List, Literal, Mapping, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Tuple, Union from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -1431,27 +1430,13 @@ def _parse(line: str, filename: str) -> DebianRepository: else: raise InvalidSourceError("An invalid sources line was found in %s!", filename) - @typing.overload - def add( # pyright: ignore[reportOverlappingOverload] - self, - repo: DebianRepository, - default_filename: Optional[bool] = False, - update_cache: Literal[False] = False, - ) -> None: ... - @typing.overload - def add( - self, - repo: DebianRepository, - default_filename: Optional[bool] = False, - update_cache: Literal[True] = True, - ) -> "RepositoryMapping": ... def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented self, repo: DebianRepository, default_filename: Optional[bool] = False, update_cache: bool = False, - ) -> Optional["RepositoryMapping"]: - """Add a new repository to the system using apt-add-repository. + ) -> "RepositoryMapping": + """Add a new repository to the system using add-apt-repository. Args: repo: a DebianRepository object where repo.enabled is True @@ -1462,7 +1447,7 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und can call `update` manually before installing any packages. Returns: - None, or a new RepositoryMapping object if update_cache is True + self, or a new RepositoryMapping object if update_cache is True raises: ValueError: if repo.enabled is False @@ -1474,25 +1459,71 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und WARNING: the default_filename keyword argument is provided for backwards compatibility only. It is not used, and was not used in the previous revision of this library. """ + self._add_apt_repository(repo, update_cache=update_cache, remove=False) + if update_cache: + return RepositoryMapping() + self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo + return self + + def _remove(self, repo: DebianRepository, update_cache: bool = False) -> "RepositoryMapping": + """Use add-apt-repository to remove a repository. + + FIXME: doesn't seem to work + + Args: + repo: a DebianRepository object where repo.enabled is True + update_cache: if True, apt-add-repository will update the package cache + and then a new RepositoryMapping object will be returned. + If False, then apt-add-repository is run with the --no-update option, + and any entry for the repo is removed from this RepositoryMapping, and you + can call `update` manually before installing any packages. + + Returns: + self, or a new RepositoryMapping object if update_cache is True + """ + self._add_apt_repository(repo, update_cache=update_cache, remove=True) + if update_cache: + return RepositoryMapping() + repo_id = "-".join((repo.repotype, repo.uri, repo.release)) + self._repository_map.pop(repo_id, None) + return self + + def _add_apt_repository( + self, + repo: DebianRepository, + update_cache: bool = False, + remove: bool = False, + ) -> None: if not repo.enabled: raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) cmd = [ "apt-add-repository", - "--dry-run", "--yes", f"--uri={repo.uri}", - f"--pocket={repo.release}", ] + ## FIXME: trying to compute pocket seems like a dead end -- add by repo line format instead? + # _, _, pocket = repo.release.partition("-") + # if pocket: + # cmd.append(f"--pocket={pocket}") if repo.repotype == "deb-src": cmd.append("--enable-source") for component in repo.groups: cmd.append(f"--component={component}") if not update_cache: cmd.append("--no-update") - subprocess.run(cmd, check=True) - if update_cache: - return RepositoryMapping() - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo + if remove: + cmd.append("--remove") + logger.info(" ".join(cmd)) + try: + subprocess.run(cmd, check=True, capture_output=True) + except CalledProcessError as e: + logger.error( + "%s:\nstdout:\n%s\nstderr:\n%s", + " ".join(cmd), + e.stdout.decode(), + e.stderr.decode(), + ) + raise def disable(self, repo: DebianRepository) -> None: """Remove a repository by disabling it in the source file. From 2b726c49c8819e079a5045996bef829b6d61df19 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:07:56 +1300 Subject: [PATCH 30/50] feat: allow _Deb822Stanza from an empty set of lines --- lib/charms/operator_libs_linux/v0/apt.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index a2e258c0..f6780182 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1543,11 +1543,16 @@ class _Deb822Stanza: """ def __init__(self, numbered_lines: List[Tuple[int, str]], filename: str = ""): - self._numbered_lines = numbered_lines self._filename = filename - self._options, self._line_numbers = _deb822_stanza_to_options(numbered_lines) + self._numbered_lines = numbered_lines + if not numbered_lines: + self._repositories = () + self._gpg_key_filename = "" + self._gpg_key_from_stanza = None + return + options, line_numbers = _deb822_stanza_to_options(numbered_lines) repos, gpg_key_info = _deb822_options_to_repos( - self._options, line_numbers=self._line_numbers, filename=filename + options, line_numbers=line_numbers, filename=filename ) for repo in repos: repo._deb822_stanza = self # pyright: ignore[reportPrivateUsage] From 9d3230ac590c6bbae7b1288f40f8920763447d83 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:10:36 +1300 Subject: [PATCH 31/50] tests: add unit tests that use files on disk --- .../data/fake_apt_dir_bionic/sources.list | 42 +++++++++ .../unit/data/fake_apt_dir_noble/sources.list | 1 + .../inkscape_dev-ubuntu-stable-noble.sources | 34 +++++++ .../sources.list.d/ubuntu.sources | 11 +++ .../sources.list | 1 + .../sources.list.d/ubuntu.sources | 1 + .../sources.list | 1 + .../sources.list | 1 + .../inkscape_dev-ubuntu-stable-noble.sources | 34 +++++++ .../sources.list.d/ubuntu.sources | 11 +++ tests/unit/test_deb822.py | 91 +++++++++++++++++-- 11 files changed, 220 insertions(+), 8 deletions(-) create mode 100755 tests/unit/data/fake_apt_dir_bionic/sources.list create mode 100644 tests/unit/data/fake_apt_dir_noble/sources.list create mode 100644 tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources create mode 100644 tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources create mode 100644 tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list create mode 100644 tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources create mode 100644 tests/unit/data/fake_apt_dir_noble_no_sources/sources.list create mode 100644 tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list create mode 100644 tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources create mode 100644 tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_bionic/sources.list b/tests/unit/data/fake_apt_dir_bionic/sources.list new file mode 100755 index 00000000..5fdc50fb --- /dev/null +++ b/tests/unit/data/fake_apt_dir_bionic/sources.list @@ -0,0 +1,42 @@ +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://nz.archive.ubuntu.com/ubuntu/ bionic main restricted +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb http://nz.archive.ubuntu.com/ubuntu/ bionic-updates main restricted +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb http://nz.archive.ubuntu.com/ubuntu/ bionic universe +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic universe +deb http://nz.archive.ubuntu.com/ubuntu/ bionic-updates universe +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb http://nz.archive.ubuntu.com/ubuntu/ bionic multiverse +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic multiverse +deb http://nz.archive.ubuntu.com/ubuntu/ bionic-updates multiverse +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb http://nz.archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse +deb-src http://nz.archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu bionic partner +# deb-src http://archive.canonical.com/ubuntu bionic partner \ No newline at end of file diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list b/tests/unit/data/fake_apt_dir_noble/sources.list new file mode 100644 index 00000000..eb39b946 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble/sources.list @@ -0,0 +1 @@ +# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources b/tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources new file mode 100644 index 00000000..9bce7f16 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources @@ -0,0 +1,34 @@ +Types: deb +URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ +Suites: noble +Components: main +Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources b/tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources new file mode 100644 index 00000000..06c19eae --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources @@ -0,0 +1,11 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg diff --git a/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list b/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list new file mode 100644 index 00000000..eb39b946 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list @@ -0,0 +1 @@ +# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources b/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources new file mode 100644 index 00000000..53bc2b4d --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources @@ -0,0 +1 @@ +# this is a bad file \ No newline at end of file diff --git a/tests/unit/data/fake_apt_dir_noble_no_sources/sources.list b/tests/unit/data/fake_apt_dir_noble_no_sources/sources.list new file mode 100644 index 00000000..eb39b946 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_no_sources/sources.list @@ -0,0 +1 @@ +# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list new file mode 100644 index 00000000..eb39b946 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list @@ -0,0 +1 @@ +# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources new file mode 100644 index 00000000..9bce7f16 --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources @@ -0,0 +1,34 @@ +Types: deb +URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ +Suites: noble +Components: main +Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources new file mode 100644 index 00000000..06c19eae --- /dev/null +++ b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources @@ -0,0 +1,11 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 8d093261..9abefe3e 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -5,10 +5,14 @@ import typing import unittest +from pathlib import Path from unittest.mock import ANY, mock_open, patch from charms.operator_libs_linux.v0 import apt +UNIT_TEST_DIR = Path(__file__).parent + + ubuntu_sources_deb822 = """ Types: deb URIs: http://nz.archive.ubuntu.com/ubuntu/ @@ -201,16 +205,67 @@ def test_parse_deb822_lines_w_comments(self): [error] = errors assert isinstance(error, apt.InvalidSourceError) - def test_load_deb822_ubuntu_sources(self): - def isnt_file(f: str) -> bool: - return False + def test_init_no_files(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_empty"), + ): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] + def test_init_with_good_sources_list(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_bionic"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + def test_init_with_bad_sources_list_no_fallback(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_no_sources"), + ): + with self.assertRaises(apt.InvalidSourceError): + apt.RepositoryMapping() + + def test_init_with_bad_sources_list_fallback_ok(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_noble"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + def test_init_with_bad_ubuntu_sources(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_empty_sources"), + ): + with self.assertRaises(apt.InvalidSourceError): + apt.RepositoryMapping() + + def test_init_with_third_party_inkscape_source(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_with_inkscape"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() + def test_load_deb822_ubuntu_sources(self): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(UNIT_TEST_DIR / "data/fake_apt_dir_empty"), + ): + repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): @@ -297,3 +352,23 @@ def iglob_sources(s: str) -> typing.Iterable[str]: == sources_repo.options # pyright: ignore[reportUnknownMemberType] ) + def test_disable_with_deb822(self): + def isnt_file(f: str) -> bool: + return False + + def iglob_nothing(s: str) -> typing.Iterable[str]: + return [] + + with patch("os.path.isfile", new=isnt_file): + with patch("glob.iglob", new=iglob_nothing): + repository_mapping = apt.RepositoryMapping() + repo = apt.DebianRepository( + enabled=True, + repotype="deb", + uri="http://nz.archive.ubuntu.com/ubuntu/", + release="noble", + groups=["main", "restricted"], + ) + repo._deb822_stanza = apt._Deb822Stanza(numbered_lines=[]) + with self.assertRaises(NotImplementedError): + repository_mapping.disable(repo) From 620cb150eb76329a08a05bd7a1e75805dffb77fe Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 13:12:16 +1300 Subject: [PATCH 32/50] tests: update integration tests --- tests/integration/test_apt.py | 198 ++++++++++++++++++++++++---------- 1 file changed, 141 insertions(+), 57 deletions(-) diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index 90c5147b..fd76595c 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -4,38 +4,64 @@ import logging +import subprocess from urllib.request import urlopen from charms.operator_libs_linux.v0 import apt from helpers import get_command_path logger = logging.getLogger(__name__) - - -def test_install_package(): - try: - apt.update() - apt.add_package("zsh") - apt.add_package(["golang-cfssl", "jq"]) - except apt.PackageNotFoundError: - logger.error("A specified package not found in package cache or on system") - except apt.PackageError as e: - logger.error("Could not install package. Reason: {}".format(e.message)) - +INKSCAPE_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- +""" + + +def test_install_packages(): + apt.update() + apt.add_package("zsh") assert get_command_path("zsh") == "/usr/bin/zsh" + apt.add_package(["golang-cfssl", "jq"]) assert get_command_path("cfssl") == "/usr/bin/cfssl" assert get_command_path("jq") == "/usr/bin/jq" def test_install_package_error(): + package = apt.DebianPackage( + name="ceci-n'est-pas-un-paquet", + version="1.0", + epoch="", + arch="amd64", + state=apt.PackageState.Available, + ) try: - package = apt.DebianPackage( - "ceci-n'est-pas-un-paquet", - "1.0", - "", - "amd64", - apt.PackageState.Available, - ) package.ensure(apt.PackageState.Present) except apt.PackageError as e: assert "Unable to locate package" in str(e) @@ -44,53 +70,111 @@ def test_install_package_error(): def test_remove_package(): # First ensure the package is present cfssl = apt.DebianPackage.from_apt_cache("golang-cfssl") - if not cfssl.present: - apt.add_package("golang-cfssl") - assert get_command_path("cfssl") == "/usr/bin/cfssl" + assert not cfssl.present + # Add package + apt.add_package("golang-cfssl") + assert get_command_path("cfssl") + subprocess.run(["cfssl", "version"], check=True) # Now remove the package and check its bins disappear too apt.remove_package("golang-cfssl") - assert get_command_path("cfssl") == "" - + assert not get_command_path("cfssl") -def test_install_package_external_repository(): - repositories = apt.RepositoryMapping() - # Get the Hashicorp GPG key - key = urlopen("https://apt.releases.hashicorp.com/gpg").read().decode() +def test_install_hardware_observer_ssacli(): + _line = "deb http://downloads.linux.hpe.com/SDR/repo/mcp stretch/current non-free" - # Add the hashicorp repository if it doesn't already exist - if "deb-apt.releases.hashicorp.com-focal" not in repositories: - line = "deb [arch=amd64] https://apt.releases.hashicorp.com focal main" - repo = apt.DebianRepository.from_repo_line(line) - # Import the repository's key - repo.import_key(key) - repositories.add(repo) - apt.update() - apt.add_package("terraform") - - assert get_command_path("terraform") == "/usr/local/bin/terraform" - - -def test_list_file_generation_external_repository(): - repositories = apt.RepositoryMapping() - - # Get the mongo GPG key - key = urlopen(" https://www.mongodb.org/static/pgp/server-5.0.asc").read().decode() - - # Add the mongo repository if it doesn't already exist - repo_name = "deb-https://repo.mongodb.org/apt/ubuntu-focal/mongodb-org/5.0" - if repo_name not in repositories: - line = "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" - repo = apt.DebianRepository.from_repo_line(line) - # Import the repository's key - repo.import_key(key) - repositories.add(repo) +def test_install_package_from_external_repository(): + repo_id = "deb-https://repo.mongodb.org/apt/ubuntu-jammy/mongodb-org/8.0" + repos_before = apt.RepositoryMapping() + assert repo_id not in repos_before + assert not get_command_path("mongod") + key = urlopen("https://www.mongodb.org/static/pgp/server-8.0.asc").read().decode() + line = "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/8.0 multiverse" + ## this is one valid order of operations: + ## apt.import_key(...) # before add + ## apt.RepositoryMapping.add(repo, update_cache=True) + ## apt.add_package(...) + apt.import_key(key) + ## if: use implicit write_file + _repo = apt.DebianRepository.from_repo_line(line) apt.update() + repos_after = apt.RepositoryMapping() + ## if: don't use implicit write_file + # repo = apt.DebianRepository.from_repo_line(line, write_file=False) + # repos_after = repos_before.add(repo, update_cache=True) + ## fi + assert repo_id in repos_after apt.add_package("mongodb-org") - - assert get_command_path("mongod") == "/usr/bin/mongod" + assert get_command_path("mongod") + subprocess.run(["mongod", "--version"], check=True) + + ## cleanup + apt.remove_package("mongodb-org", autoremove=True) # mongodb-org is a metapackage + assert not get_command_path("mongod") + ## FIXME: RepositoryMapping._remove + # repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] + # assert repo_id not in repos_after + # apt.update() + # repos_clean = apt.RepositoryMapping() + # assert repo_id not in repos_clean + + +def test_install_higher_version_package_from_external_repository(): + repo_id = "deb-https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/-jammy" + repos_before = apt.RepositoryMapping() + assert repo_id not in repos_before + + # version before + if not get_command_path("inkscape"): + apt.add_package("inkscape") + version_before = subprocess.run( + ["inkscape", "--version"], + capture_output=True, + check=True, + text=True, + ).stdout + apt.remove_package("inkscape") + assert not get_command_path("inkscape") + + repo = apt.DebianRepository( + enabled=True, + repotype="deb", + uri="https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/", + release="jammy", + groups=["main"], + ) + ## this is a different, valid order of operations: + ## apt.RepositoryMapping.add(..., update_cache=False) + ## apt.import_key(...) # before update but after add + ## apt.update() + ## apt.add_package(...) + repos_after = repos_before.add(repo, update_cache=False) # default update_cache option + assert repo_id in repos_after + apt.import_key(INKSCAPE_KEY) + apt.update() + apt.add_package("inkscape") + assert get_command_path("inkscape") + version_after = subprocess.run( + ["inkscape", "--version"], + capture_output=True, + check=True, + text=True, + ).stdout + assert version_after > version_before # lexical comparison :( + + ## cleanup + apt.remove_package("inkscape") + assert not get_command_path("inkscape") + ## FIXME: RepositoryMapping._remove + # repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] + # assert repo_id not in repos_after + # apt.update() + # repos_clean = apt.RepositoryMapping() + # assert repo_id not in repos_clean + # repos_clean = repos_after._remove(repo, update_cache=True) # pyright: ignore[reportPrivateUsage] + # assert repo_id not in repos_clean def test_from_apt_cache_error(): From eafe86941ca8ab3c2cd513df38e68df495df5d3a Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 14:33:23 +1300 Subject: [PATCH 33/50] feat: use sourceslist (one-per-line) format for add-apt-repository --- lib/charms/operator_libs_linux/v0/apt.py | 116 ++++++++++++----------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index f6780182..20f7c98b 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -941,8 +941,8 @@ def __init__( uri: str, release: str, groups: List[str], - filename: Optional[str] = "", - gpg_key_filename: Optional[str] = "", + filename: str = "", + gpg_key_filename: str = "", options: Optional[Dict[str, str]] = None, ): self._enabled = enabled @@ -954,6 +954,24 @@ def __init__( self._gpg_key_filename = gpg_key_filename self._options = options + def _get_identifier(self) -> str: + """Return str identifier derived from repotype, uri, and release. + + Private method used to produce the identifiers used by RepositoryMapping. + """ + return "{}-{}-{}".format(self.repotype, self.uri, self.release) + + def _to_line(self) -> str: + """Return the one-per-line format repository definition.""" + return "{prefix}{repotype} {options}{uri} {release} {groups}".format( + prefix="" if self.enabled else "#", + repotype=self.repotype, + options=self.make_options_string(), + uri=self.uri, + release=self.release, + groups=" ".join(self.groups), + ) + @property def enabled(self): """Return whether or not the repository is enabled.""" @@ -995,6 +1013,16 @@ def filename(self, fname: str) -> None: raise InvalidSourceError("apt source filenames should end in .list or .sources!") self._filename = fname + def _make_filename(self) -> str: + """Construct a filename from uri and release. + + For internal use when a filename isn't set. + """ + return "{}-{}.list".format( + DebianRepository.prefix_from_uri(self.uri), + self.release.replace("/", "-"), + ) + @property def gpg_key(self): """Returns the path to the GPG key for this repository.""" @@ -1022,7 +1050,8 @@ def make_options_string(self) -> str: if not options: return "" - return "[{}] ".format(" ".join("{}={}".format(k, v) for k, v in options.items())) + pairs = ("{}={}".format(k, v) for k, v in sorted(options.items())) + return "[{}] ".format(" ".join(pairs)) @staticmethod def prefix_from_uri(uri: str) -> str: @@ -1041,38 +1070,19 @@ def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "Debian repo_line: a string representing a repository entry write_file: boolean to enable writing the new repo to disk """ - repo = RepositoryMapping._parse(repo_line, "UserInput") - fname = "{}-{}.list".format( - DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") - ) - repo.filename = fname - - options = repo.options if repo.options else {} - if repo.gpg_key: - options["signed-by"] = repo.gpg_key - - # For Python 3.5 it's required to use sorted in the options dict in order to not have - # different results in the order of the options between executions. - options_str = ( - "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in sorted(options.items())])) - if options - else "" - ) - - repo_str = ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, options_str, repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) + repo = RepositoryMapping._parse( # pyright: ignore[reportPrivateUsage] + repo_line, "UserInput" ) - + repo.filename = repo._make_filename() + repo_str = repo._to_line() + "\n" if write_file: logger.info( "DebianRepository.from_repo_line('%s', write_file=True)\nWriting to '%s':\n%s", repo_line, - fname, + repo.filename, repo_str, ) - with open(fname, "wb") as f: + with open(repo.filename, "wb") as f: f.write(repo_str.encode("utf-8")) return repo @@ -1307,8 +1317,8 @@ def load_deb822(self, filename: str) -> None: repos, errors = self._parse_deb822_lines(f, filename=filename) for repo in repos: - repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) - self._repository_map[repo_identifier] = repo + identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] + self._repository_map[identifier] = repo if errors: logger.debug( @@ -1442,9 +1452,9 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und repo: a DebianRepository object where repo.enabled is True update_cache: if True, apt-add-repository will update the package cache and then a new RepositoryMapping object will be returned. - If False, then apt-add-repository is run with the --no-update option, - and an entry for repo is added to this RepositoryMapping, and you - can call `update` manually before installing any packages. + If False, then apt-add-repository is run with the --no-update option. + An entry for the repo is added to this RepositoryMapping before returning it. + Don't forget to call `apt.update` manually before installing any packages! Returns: self, or a new RepositoryMapping object if update_cache is True @@ -1462,30 +1472,33 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und self._add_apt_repository(repo, update_cache=update_cache, remove=False) if update_cache: return RepositoryMapping() - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo + identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] + self._repository_map[identifier] = repo return self def _remove(self, repo: DebianRepository, update_cache: bool = False) -> "RepositoryMapping": """Use add-apt-repository to remove a repository. - FIXME: doesn't seem to work - Args: repo: a DebianRepository object where repo.enabled is True update_cache: if True, apt-add-repository will update the package cache and then a new RepositoryMapping object will be returned. - If False, then apt-add-repository is run with the --no-update option, - and any entry for the repo is removed from this RepositoryMapping, and you - can call `update` manually before installing any packages. + If False, then apt-add-repository is run with the --no-update option. + Any entry for the repo is removed from this RepositoryMapping before returning it. + Don't forget to call `apt.update` manually before installing any packages! Returns: self, or a new RepositoryMapping object if update_cache is True + + raises: + ValueError: if repo.enabled is False + CalledProcessError: if there's an error running apt-add-repository """ self._add_apt_repository(repo, update_cache=update_cache, remove=True) if update_cache: return RepositoryMapping() - repo_id = "-".join((repo.repotype, repo.uri, repo.release)) - self._repository_map.pop(repo_id, None) + identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] + self._repository_map.pop(identifier, None) return self def _add_apt_repository( @@ -1496,30 +1509,23 @@ def _add_apt_repository( ) -> None: if not repo.enabled: raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) + line = repo._to_line() # pyright: ignore[reportPrivateUsage] cmd = [ "apt-add-repository", "--yes", - f"--uri={repo.uri}", + "--sourceslist=" + line, ] - ## FIXME: trying to compute pocket seems like a dead end -- add by repo line format instead? - # _, _, pocket = repo.release.partition("-") - # if pocket: - # cmd.append(f"--pocket={pocket}") - if repo.repotype == "deb-src": - cmd.append("--enable-source") - for component in repo.groups: - cmd.append(f"--component={component}") - if not update_cache: - cmd.append("--no-update") if remove: cmd.append("--remove") - logger.info(" ".join(cmd)) + if not update_cache: + cmd.append("--no-update") + logger.info("%s", cmd) try: subprocess.run(cmd, check=True, capture_output=True) except CalledProcessError as e: logger.error( - "%s:\nstdout:\n%s\nstderr:\n%s", - " ".join(cmd), + "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", + cmd, e.stdout.decode(), e.stderr.decode(), ) From 29457656c7f21c0c95b0e7d22fc350f56802bca3 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 14:46:56 +1300 Subject: [PATCH 34/50] feat: call _add_apt_repository in from_repo_line (don't reimplement logic) --- lib/charms/operator_libs_linux/v0/apt.py | 19 +++++++------------ tests/unit/test_repo.py | 20 +++++++++++++------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 20f7c98b..4d25805a 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1017,6 +1017,7 @@ def _make_filename(self) -> str: """Construct a filename from uri and release. For internal use when a filename isn't set. + Should match the filename written to by add-apt-repository. """ return "{}-{}.list".format( DebianRepository.prefix_from_uri(self.uri), @@ -1068,23 +1069,17 @@ def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "Debian Args: repo_line: a string representing a repository entry - write_file: boolean to enable writing the new repo to disk + write_file: boolean to enable writing the new repo to disk. True by default. + Results in calling `add-apt-repository --no-update --sourceslist $repo_line` """ repo = RepositoryMapping._parse( # pyright: ignore[reportPrivateUsage] - repo_line, "UserInput" + repo_line, filename="UserInput" # temp filename ) repo.filename = repo._make_filename() - repo_str = repo._to_line() + "\n" if write_file: - logger.info( - "DebianRepository.from_repo_line('%s', write_file=True)\nWriting to '%s':\n%s", - repo_line, - repo.filename, - repo_str, + RepositoryMapping._add_apt_repository( # pyright: ignore[reportPrivateUsage]) + repo, update_cache=False ) - with open(repo.filename, "wb") as f: - f.write(repo_str.encode("utf-8")) - return repo def disable(self) -> None: @@ -1501,8 +1496,8 @@ def _remove(self, repo: DebianRepository, update_cache: bool = False) -> "Reposi self._repository_map.pop(identifier, None) return self + @staticmethod def _add_apt_repository( - self, repo: DebianRepository, update_cache: bool = False, remove: bool = False, diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index 29061262..dc4c6e1b 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -107,14 +107,18 @@ def test_can_add_repositories(self): ) def test_can_add_repositories_from_string(self): - d = apt.DebianRepository.from_repo_line("deb https://example.com/foo focal bar baz") + d = apt.DebianRepository.from_repo_line( + "deb https://example.com/foo focal bar baz", + write_file=False, + ) self.assertEqual(d.enabled, True) self.assertEqual(d.repotype, "deb") self.assertEqual(d.uri, "https://example.com/foo") self.assertEqual(d.release, "focal") self.assertEqual(d.groups, ["bar", "baz"]) self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") - self.assertIn("deb https://example.com/foo focal bar baz\n", open(d.filename).readlines()) + ## FIXME: need integration test for write_file=True + # self.assertIn("deb https://example.com/foo focal bar baz\n", open(d.filename).readlines()) @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") def test_valid_list_file(self): @@ -135,7 +139,8 @@ def test_valid_list_file(self): def test_can_add_repositories_from_string_with_options(self): d = apt.DebianRepository.from_repo_line( - "deb [signed-by=/foo/gpg.key arch=amd64] https://example.com/foo focal bar baz" + "deb [signed-by=/foo/gpg.key arch=amd64] https://example.com/foo focal bar baz", + write_file=False, ) self.assertEqual(d.enabled, True) self.assertEqual(d.repotype, "deb") @@ -145,7 +150,8 @@ def test_can_add_repositories_from_string_with_options(self): self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") self.assertEqual(d.gpg_key, "/foo/gpg.key") self.assertEqual(d.options["arch"], "amd64") - self.assertIn( - "deb [arch=amd64 signed-by=/foo/gpg.key] https://example.com/foo focal bar baz\n", - open(d.filename).readlines(), - ) + ## FIXME: need integration test for write_file=True + # self.assertIn( + # "deb [arch=amd64 signed-by=/foo/gpg.key] https://example.com/foo focal bar baz\n", + # open(d.filename).readlines(), + # ) From 12321750288ebe9586e2189e98750fdf4c9e6b18 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 15:00:16 +1300 Subject: [PATCH 35/50] tests: re-enable package cleanup in integration tests --- tests/integration/test_apt.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index fd76595c..194b78cf 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -98,7 +98,7 @@ def test_install_package_from_external_repository(): ## apt.add_package(...) apt.import_key(key) ## if: use implicit write_file - _repo = apt.DebianRepository.from_repo_line(line) + repo = apt.DebianRepository.from_repo_line(line) apt.update() repos_after = apt.RepositoryMapping() ## if: don't use implicit write_file @@ -113,12 +113,11 @@ def test_install_package_from_external_repository(): ## cleanup apt.remove_package("mongodb-org", autoremove=True) # mongodb-org is a metapackage assert not get_command_path("mongod") - ## FIXME: RepositoryMapping._remove - # repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] - # assert repo_id not in repos_after - # apt.update() - # repos_clean = apt.RepositoryMapping() - # assert repo_id not in repos_clean + repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] + assert repo_id not in repos_after + apt.update() + repos_clean = apt.RepositoryMapping() + assert repo_id not in repos_clean def test_install_higher_version_package_from_external_repository(): @@ -167,14 +166,10 @@ def test_install_higher_version_package_from_external_repository(): ## cleanup apt.remove_package("inkscape") assert not get_command_path("inkscape") - ## FIXME: RepositoryMapping._remove - # repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] - # assert repo_id not in repos_after - # apt.update() - # repos_clean = apt.RepositoryMapping() - # assert repo_id not in repos_clean - # repos_clean = repos_after._remove(repo, update_cache=True) # pyright: ignore[reportPrivateUsage] - # assert repo_id not in repos_clean + repos_clean = repos_after._remove( # pyright: ignore[reportPrivateUsage] + repo, update_cache=True + ) + assert repo_id not in repos_clean def test_from_apt_cache_error(): From 569c39e0a494e6054e30adcf128b743a7344b48b Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 16:38:43 +1300 Subject: [PATCH 36/50] tests: add test case from hardware-observer charm and cleanup keys --- tests/integration/test_apt.py | 241 ++++++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 40 deletions(-) diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index 194b78cf..8baa0518 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -4,44 +4,15 @@ import logging +import os import subprocess +from typing import List from urllib.request import urlopen from charms.operator_libs_linux.v0 import apt from helpers import get_command_path logger = logging.getLogger(__name__) -INKSCAPE_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- - . - mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU - 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM - 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 - vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U - NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu - 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP - pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww - 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e - ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA - RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ - S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB - tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 - FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC - AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle - amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO - qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu - oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R - PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE - MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom - 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 - J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ - q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z - lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k - uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO - 2ekz6IVTXVA= - =VF33 - -----END PGP PUBLIC KEY BLOCK----- -""" def test_install_packages(): @@ -81,7 +52,53 @@ def test_remove_package(): def test_install_hardware_observer_ssacli(): - _line = "deb http://downloads.linux.hpe.com/SDR/repo/mcp stretch/current non-free" + """Test the ability to install a package used by the hardware-observer charm. + + Here we follow the order of operations and arguments used in the charm: + for key in HP_KEYS: + apt.import_key(key) + + repositories = apt.RepositoryMapping() + repo = apt.DebianRepository.from_repo_line(...) + repositories.add(repo) + + apt.add_package(self.pkg, update_cache=True) + """ + line = "deb http://downloads.linux.hpe.com/SDR/repo/mcp stretch/current non-free" + repo_id = apt.DebianRepository.from_repo_line( + line, write_file=False + )._get_identifier() # pyright: ignore[reportPrivateUsage] + repos_before = apt.RepositoryMapping() + assert repo_id not in repos_before + assert not get_command_path("ssacli") + + key_files: List[str] = [] # just for cleanup + ## begin steps + for key in HP_KEYS: + key_file = apt.import_key(key) + key_files.append(key_file) + repositories = apt.RepositoryMapping() + repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default + # repo added to system but repositories doesn't know about it yet + assert repo_id not in repositories + repositories.add(repo) # update_cache=False by default + # `add` call is redundant with `from_repo_line` from system pov + # but it does add an entry to the RepositoryMapping + assert repo_id in repositories + apt.add_package("ssacli", update_cache=True) + assert get_command_path("ssacli") + # install succeed here as update_cache=True + ## end steps + + ## cleanup + for key_file in key_files: + os.remove(key_file) + repos_clean = repositories._remove( # pyright: ignore[reportPrivateUsage] + repo, update_cache=True + ) + assert repo_id not in repos_clean + apt.remove_package("ssacli") + assert not get_command_path("ssacli") def test_install_package_from_external_repository(): @@ -96,9 +113,9 @@ def test_install_package_from_external_repository(): ## apt.import_key(...) # before add ## apt.RepositoryMapping.add(repo, update_cache=True) ## apt.add_package(...) - apt.import_key(key) + key_file = apt.import_key(key) ## if: use implicit write_file - repo = apt.DebianRepository.from_repo_line(line) + repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default apt.update() repos_after = apt.RepositoryMapping() ## if: don't use implicit write_file @@ -111,13 +128,16 @@ def test_install_package_from_external_repository(): subprocess.run(["mongod", "--version"], check=True) ## cleanup - apt.remove_package("mongodb-org", autoremove=True) # mongodb-org is a metapackage - assert not get_command_path("mongod") + os.remove(key_file) repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] assert repo_id not in repos_after + repos_before_update = apt.RepositoryMapping() + assert repo_id in repos_before_update # update hasn't been called yet! apt.update() repos_clean = apt.RepositoryMapping() assert repo_id not in repos_clean + apt.remove_package("mongodb-org", autoremove=True) # mongodb-org is a metapackage + assert not get_command_path("mongod") def test_install_higher_version_package_from_external_repository(): @@ -149,9 +169,9 @@ def test_install_higher_version_package_from_external_repository(): ## apt.import_key(...) # before update but after add ## apt.update() ## apt.add_package(...) - repos_after = repos_before.add(repo, update_cache=False) # default update_cache option + repos_after = repos_before.add(repo) # update_cache=False by default assert repo_id in repos_after - apt.import_key(INKSCAPE_KEY) + key_file = apt.import_key(INKSCAPE_KEY) apt.update() apt.add_package("inkscape") assert get_command_path("inkscape") @@ -164,12 +184,13 @@ def test_install_higher_version_package_from_external_repository(): assert version_after > version_before # lexical comparison :( ## cleanup - apt.remove_package("inkscape") - assert not get_command_path("inkscape") + os.remove(key_file) repos_clean = repos_after._remove( # pyright: ignore[reportPrivateUsage] repo, update_cache=True ) assert repo_id not in repos_clean + apt.remove_package("inkscape") + assert not get_command_path("inkscape") def test_from_apt_cache_error(): @@ -177,3 +198,143 @@ def test_from_apt_cache_error(): apt.DebianPackage.from_apt_cache("ceci-n'est-pas-un-paquet") except apt.PackageError as e: assert "No packages found" in str(e) + + +INKSCAPE_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- +""" +"""Static parameters for keys.""" + +HPPUBLICKEY1024 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.0 (MingW32) + +mQGiBEIxWpoRBADb06sJgnD7MJnm2Ny1nmTFLDSZ8vkubP+pmfn9N9TE26oit+KI +OnVTRVbSPl3F15wTjSBGR453MEfnzp1NrMk1GIa/m1nKAmgQ4t1714C4jQab0to+ +gP51XhPhtAGt7BggorQw2RXa4KdTCh8ByOIaDKRYcESmMazSZ+Pscy2XRwCgm771 +21RCM0RcG2dmHZZgKH8fTscD/RiY3CHI2jJl9WosIYXbZpOySzrLn0lRCRdNdpew +Y5m1f3lhqoSvJk7pXjs4U+3XlOlUhgWl5HiXuWSVyPu2ilfGdfgpJslawI85fBQg +Ul5kcrjLHHsApeG8oGStFJE2JAc+0D+whmGmJbjWKwuZJmgpm9INplA4h1BYJbx+ +6A3MBACFiMTttDPpJ+5eWr1VSZwxCZNqvPWmjpL5Nh9F8xzE7q+ad2CFKSebvRrv +Jf7Y2m+wY9bmo5nJ3wHYEX3Aatt+QVF10G6wTdIz/Ohm/Pc4Li4NhzYOv7FKxVam +97UN0O8Rsl4GhE2eE8H+Q3QYFvknAWoTj3Rq3/A5FA6FsRFhxbQwSGV3bGV0dC1Q +YWNrYXJkIENvbXBhbnkgKEhQIENvZGVzaWduaW5nIFNlcnZpY2UpiGQEExECACQF +AkIxWpoCGwMFCRLMAwAGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQUnvFOiaJuIc1 +2wCgj2UotUgSegPHmcKdApY+4WFaz/QAnjI58l5bDD8eElBCErHVoq9uPMczuQIN +BEIxWqUQCADnBXqoU8QeZPEy38oI0GrN2q7nvS+4UBQeIRVy8x+cOqDRDcE8PHej +7NtxP698U0WFGK47GszjiV4WTnvexuJk0B5AMEBHana8fVj7uRUcmyYZqOZd7EXn +Q3Ivi8itfkTICkhZi7bmGsSF0iJ0eAI5n2bCqJykNQvJ6a3dWJKP8EgaBCZj+TGL +WWJHDZsrn8g4BeaNS/MbmsCLAk8N6bWMGzAKfgxUraMCwuZ9fVyHFavHdeChUtna +qnF4uw0hHLaGWmTJjziXVvVC1a8+inTxPZkVpAvD0A+/LNlkP7TtAdaVOJqv3+a3 +ybMQL851bRTFyt+H0XGHhzhhtuu9+DyfAAMFCADRWGxIfniVG7O4wtwLD3sWzR/W +LmFlJYu4s9rSDgn3NDjigQzZoVtbuv3Z9IZxBMoYa50MuybuVDp55z/wmxvYoW2G +25kOFDKx/UmkKkUBLdokb5V1p9j5SJorGBSfsNAHflhmBhyuMP4CDISbBUSN7oO1 +Oj41jNxpqhy+8ayygSVcTNwMe909J/HdC//xFANLDhjKPf3ZAulWNhOvjTlpF46B +yt1l8ZNinIeE7CFL7H+LlMl2Ml6wsOkrxsSauBis6nER4sYVqrMdzpUU2Sr2hj6Q +sJ+9TS+IURcnxL/M851KCwLhwZKdphQjT3mXXsoCx/l3rI6cxpwYgjiKiZhOiE8E +GBECAA8FAkIxWqUCGwwFCRLMAwAACgkQUnvFOiaJuIenewCdHcEvMxBYprqRjKUw +04EypyFtZTgAn0wds0nbpd2+VZ5WHbVRfU4y5Y5Y +=+cX+ +-----END PGP PUBLIC KEY BLOCK----- +""" + +HPPUBLICKEY2048 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (MingW32) + +mQENBFC+QboBCAC1bodHD7AmR00SkDMB4u9MXy+Z5vv8wbmGRaKDBYScpAknOljX +d5tBADffAetd1hgLnrLKN8vHdIsYkmUyeEeEsnIUKtwvbx/f6PoZZPOIIIRh1d2W +Mjw9qXIE+tgr2gWlq0Gi5BZzaKse1+khRQ2rewJBppblSGWgcmCMIq8OwAsrdbtr +z7+37c/g/Y2VfAahc23YZW9LQ5MiaI4nS4JMZbWPYtBdF78B/D2t5FvmvDG0Cgjk +Qi1U9IVjiFKixuoi6nRsvBLFYL/cI+vo4iyUC5x7qmKd8gN7A030gS67VrleNRki +q0vaF6J46XpIl4o58t23FSAKKRbTwavYzdMpABEBAAG0NEhld2xldHQtUGFja2Fy +ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSmJAT4EEwECACgF +AlC+QboCGwMFCRLMAwAGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELBwaApc +4tR2x7sH/A3D4XxEEyrX6Z3HeWSSA80+n+r5QwfXm5unxsWEL3JyNg6sojlrJY4K +8k4ih4nkY4iblChTCSQwnqKXqkL5U+RIr+AJoPx+55M98u4eRTVYMHZD7/jFq85z +ZFGUkFkars9E2aRzWhqbz0LINb9OUeX0tT5qQseHflO2PaJykxNPC14WhsBKC2lg +dZWnGhO5QJFp69AnSp4k+Uo/1LMk87YEJIL1NDR0lrlKgRvFfFyTpRBt+Qb1Bb7g +rjN0171g8t5GaPWamN3Oua/v4aZg15f3xydRF8y9TsYjiNz+2TzRjKv7AkpZaJST +06CqMjCgiZ6UFFGN0/oqLnwxdP3Mmh4= +=aphN +-----END PGP PUBLIC KEY BLOCK----- +""" + +HPPUBLICKEY2048_KEY1 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (MingW32) + +mQENBFRtGAgBCADlSku65P14hVdx9E/W0n6MwuB3WGqmsyKNoa3HezFdMjWERldI +NNUdi8O28cZ6j2+Hi9L1HeQIQ9+7FHpR3JyQePBJtRX8WSEusfRtML98opDhJxKm +8Jyxb7aTvCwdNHz3yxADINkMtOj5oRm7VCr8XHkG7YU27ELs8B+BXWvjO21oSosi +FurnhT+H3hQsYXfYA55aa21q0qX+L5dFJSNdzZVo7m9ybioVv2R5+PfBvdaSxCnm +OpcGXFaKAsqVHeTW0pd3sdkin1rkbhOBaU5lFBt2ZiMtKpKHpT8TZnqHpFHFbgi8 +j2ARJj4IDct2OGILddUIZSFyue6WE2hpV5c/ABEBAAG0OEhld2xldHQtUGFja2Fy +ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSkgLSAxiQE+BBMB +AgAoBQJUbRgIAhsDBQkSzAMABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRD6 +3Y1ksSdeo6BJCADOfIPPLPpIOnFK9jH4t8lLUd+RyMc+alA3uTDPUJa/ZHa6DHfh +42iaPYVEV8OG0tnbMlHmwvsZ5c1/MRMw1UbxCvD88P2qM4SUrUjQUlSCms2GLGvF +ftFXBiOJQ7/yBc9o+yoSvwPrrTxSCk4+Sqm0IfVXVzChDM9dM9YPY2Vzjd+LUaYC +3X+eSuggUDO0TmJLJd7tZdF9fVXq3lr63BZ5PY98MTCuOoeSMDa9FIUQf6vn6UUJ +MDSRZ9OzhpNJOKR+ShVRwDK6My8gtVIW1EAW2w3VQWI2UNF07aLeO8UG6nTNWA23 ++OuZkUdgQovjcq01caSefgOkmiQOx6d74CAk +=X+eo +-----END PGP PUBLIC KEY BLOCK----- +""" + +HPEPUBLICKEY2048_KEY1 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +mQENBFZp0LkBCACXajRw3b4x7G7dulNYj0hUID4BtVFq/MjEb6PHckTxGxZDoQRX +RK54tiTFA9wq3b4P3yEFnOjbjRoI0d7Ls67FADugFO+cDCtsV9yuDlaYP/U/h2nX +N0R4AdYbsVd5yr6xr+GAy66Hmx5jFH3kbC+zJpOcI0tU9hcyU7gjbxu6KQ1ypI2Q +VRKf8sRBJXgmkOlbYx35ZUMFcmVxrLJXvUuxmAVXgT9f5M3Z3rsGt/ab+/+1TFSb +RsaqHsIPE0QH8ikqW4IeDQAo1T99pCdf7FWr45KFFTo7O4AZdLMWVgqeFHaSoZxJ +307VIINsWiwQoPp0tfU5NOOOwB1Sv3x9QgFtABEBAAG0P0hld2xldHQgUGFja2Fy +ZCBFbnRlcnByaXNlIENvbXBhbnkgUlNBLTIwNDgtMjUgPHNpZ25ocEBocGUuY29t +PokBPQQTAQIAJwUCVmnQuQIbLwUJEswDAAYLCQgHAwIGFQgCCQoLAxYCAQIeAQIX +gAAKCRDCCK3eJsK3l9G+B/0ekblsBeN+xHIJ28pvo2aGb2KtWBwbT1ugI+aIS17K +UQyHZJUQH+ZeRLvosuoiQEdcGIqmOxi2hVhSCQAOV1LAonY16ACveA5DFAEBz1+a +WQyx6sOLLEAVX1VqGlBXxh3XLEUWOhlAf1gZPNtHsmURTUy2h1Lv/Yoj8KLyuK2n +DmrLOS3Ro+RqWocaJfvAgXKgt6Fq/ChDUHOnar7lGswzMsbE/yzLJ7He4y89ImK+ +2ktR5HhDuxqgCe9CWH6Q/1WGhUa0hZ3nbluq7maa+kPe2g7JcRzPH/nJuDCAOZ7U +6mHE8j0kMQMYjgaYEx2wc02aQRmPyxhbDLjSbtjomXRr +=voON +-----END PGP PUBLIC KEY BLOCK----- +""" + +HP_KEYS = ( + HPEPUBLICKEY2048_KEY1, + HPPUBLICKEY2048_KEY1, + HPPUBLICKEY2048, + HPPUBLICKEY1024, +) From b08125305084ebcd70f0760cacecf4d08d791642 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 17:36:59 +1300 Subject: [PATCH 37/50] tests: refactor deb822 unit tests --- .../bionic}/sources.list | 0 .../noble-empty-sources}/sources.list | 0 .../sources.list.d/ubuntu.sources | 0 .../noble-in-one-per-line-format/sources.list | 4 + .../noble-no-sources}/sources.list | 0 .../noble-with-comments-etc}/sources.list | 0 .../sources.list.d/ubuntu.sources | 33 +++++++ .../noble-with-inkscape}/sources.list | 0 .../inkscape_dev-ubuntu-stable-noble.sources | 0 .../sources.list.d/ubuntu.sources | 0 .../data/fake-apt-dirs/noble/sources.list | 1 + .../noble}/sources.list.d/ubuntu.sources | 0 .../inkscape_dev-ubuntu-stable-noble.sources | 34 ------- tests/unit/test_deb822.py | 96 +++++-------------- 14 files changed, 63 insertions(+), 105 deletions(-) rename tests/unit/data/{fake_apt_dir_bionic => fake-apt-dirs/bionic}/sources.list (100%) rename tests/unit/data/{fake_apt_dir_noble => fake-apt-dirs/noble-empty-sources}/sources.list (100%) rename tests/unit/data/{fake_apt_dir_noble_empty_sources => fake-apt-dirs/noble-empty-sources}/sources.list.d/ubuntu.sources (100%) create mode 100644 tests/unit/data/fake-apt-dirs/noble-in-one-per-line-format/sources.list rename tests/unit/data/{fake_apt_dir_noble_empty_sources => fake-apt-dirs/noble-no-sources}/sources.list (100%) rename tests/unit/data/{fake_apt_dir_noble_no_sources => fake-apt-dirs/noble-with-comments-etc}/sources.list (100%) create mode 100644 tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources rename tests/unit/data/{fake_apt_dir_noble_with_inkscape => fake-apt-dirs/noble-with-inkscape}/sources.list (100%) rename tests/unit/data/{fake_apt_dir_noble => fake-apt-dirs/noble-with-inkscape}/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources (100%) rename tests/unit/data/{fake_apt_dir_noble => fake-apt-dirs/noble-with-inkscape}/sources.list.d/ubuntu.sources (100%) create mode 100644 tests/unit/data/fake-apt-dirs/noble/sources.list rename tests/unit/data/{fake_apt_dir_noble_with_inkscape => fake-apt-dirs/noble}/sources.list.d/ubuntu.sources (100%) delete mode 100644 tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources diff --git a/tests/unit/data/fake_apt_dir_bionic/sources.list b/tests/unit/data/fake-apt-dirs/bionic/sources.list similarity index 100% rename from tests/unit/data/fake_apt_dir_bionic/sources.list rename to tests/unit/data/fake-apt-dirs/bionic/sources.list diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list b/tests/unit/data/fake-apt-dirs/noble-empty-sources/sources.list similarity index 100% rename from tests/unit/data/fake_apt_dir_noble/sources.list rename to tests/unit/data/fake-apt-dirs/noble-empty-sources/sources.list diff --git a/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources b/tests/unit/data/fake-apt-dirs/noble-empty-sources/sources.list.d/ubuntu.sources similarity index 100% rename from tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list.d/ubuntu.sources rename to tests/unit/data/fake-apt-dirs/noble-empty-sources/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake-apt-dirs/noble-in-one-per-line-format/sources.list b/tests/unit/data/fake-apt-dirs/noble-in-one-per-line-format/sources.list new file mode 100644 index 00000000..03481c54 --- /dev/null +++ b/tests/unit/data/fake-apt-dirs/noble-in-one-per-line-format/sources.list @@ -0,0 +1,4 @@ +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse +deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse \ No newline at end of file diff --git a/tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list b/tests/unit/data/fake-apt-dirs/noble-no-sources/sources.list similarity index 100% rename from tests/unit/data/fake_apt_dir_noble_empty_sources/sources.list rename to tests/unit/data/fake-apt-dirs/noble-no-sources/sources.list diff --git a/tests/unit/data/fake_apt_dir_noble_no_sources/sources.list b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list similarity index 100% rename from tests/unit/data/fake_apt_dir_noble_no_sources/sources.list rename to tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list diff --git a/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources new file mode 100644 index 00000000..eff19616 --- /dev/null +++ b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources @@ -0,0 +1,33 @@ +Components: main restricted universe multiverse # Components first! I don't think deb822 cares about ordering +Types: deb deb-src # this could be one or both of these options +URIs: http://nz.archive.ubuntu.com/ubuntu/ http://archive.ubuntu.com/ubuntu/ + # there can be multiple space separated URIs + # sources are specified in priority order + # apt does some de-duplication of sources after parsing too +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +# let's make this insecure! (jk, just testing parsing) +Suites: noble noble-updates noble-backports + +Foo: Bar # this is a separate (malformed) entry + +## a fully commented out stanza +#Types: deb +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +## disable security updates while we're at it + +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: an/exact/path/ + +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: an/exact/path/ +Components: main # this is an error, can't have components with an exact path + +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +# this is an error, must have at least one component if suites isn't an exact path \ No newline at end of file diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list b/tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list similarity index 100% rename from tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list rename to tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources b/tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources similarity index 100% rename from tests/unit/data/fake_apt_dir_noble/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources rename to tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources diff --git a/tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources b/tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list.d/ubuntu.sources similarity index 100% rename from tests/unit/data/fake_apt_dir_noble/sources.list.d/ubuntu.sources rename to tests/unit/data/fake-apt-dirs/noble-with-inkscape/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake-apt-dirs/noble/sources.list b/tests/unit/data/fake-apt-dirs/noble/sources.list new file mode 100644 index 00000000..eb39b946 --- /dev/null +++ b/tests/unit/data/fake-apt-dirs/noble/sources.list @@ -0,0 +1 @@ +# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources b/tests/unit/data/fake-apt-dirs/noble/sources.list.d/ubuntu.sources similarity index 100% rename from tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/ubuntu.sources rename to tests/unit/data/fake-apt-dirs/noble/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources b/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources deleted file mode 100644 index 9bce7f16..00000000 --- a/tests/unit/data/fake_apt_dir_noble_with_inkscape/sources.list.d/inkscape_dev-ubuntu-stable-noble.sources +++ /dev/null @@ -1,34 +0,0 @@ -Types: deb -URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ -Suites: noble -Components: main -Signed-By: - -----BEGIN PGP PUBLIC KEY BLOCK----- - . - mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU - 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM - 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 - vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U - NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu - 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP - pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww - 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e - ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA - RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ - S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB - tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 - FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC - AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle - amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO - qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu - oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R - PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE - MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom - 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 - J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ - q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z - lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k - uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO - 2ekz6IVTXVA= - =VF33 - -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 9abefe3e..688634ff 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -10,7 +10,8 @@ from charms.operator_libs_linux.v0 import apt -UNIT_TEST_DIR = Path(__file__).parent +TEST_DATA_DIR = Path(__file__).parent / "data" +FAKE_APT_DIRS = TEST_DATA_DIR / "fake-apt-dirs" ubuntu_sources_deb822 = """ @@ -55,43 +56,6 @@ deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse """ -inkscape_sources_deb822 = """ -Types: deb -URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ -Suites: noble -Components: main -Signed-By: - -----BEGIN PGP PUBLIC KEY BLOCK----- - . - mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU - 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM - 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 - vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U - NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu - 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP - pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww - 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e - ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA - RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ - S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB - tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 - FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC - AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle - amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO - qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu - oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R - PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE - MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom - 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 - J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ - q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z - lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k - uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO - 2ekz6IVTXVA= - =VF33 - -----END PGP PUBLIC KEY BLOCK----- -""" - class TestRepositoryMappingDeb822Behaviour(unittest.TestCase): def test_iter_deb822_paragraphs_ubuntu_sources(self): @@ -209,7 +173,7 @@ def test_init_no_files(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_empty"), + str(FAKE_APT_DIRS / "empty"), ): repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map @@ -218,7 +182,7 @@ def test_init_with_good_sources_list(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_bionic"), + str(FAKE_APT_DIRS / "bionic"), ): repository_mapping = apt.RepositoryMapping() assert repository_mapping._repository_map @@ -227,7 +191,7 @@ def test_init_with_bad_sources_list_no_fallback(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_no_sources"), + str(FAKE_APT_DIRS / "noble-no-sources"), ): with self.assertRaises(apt.InvalidSourceError): apt.RepositoryMapping() @@ -236,7 +200,7 @@ def test_init_with_bad_sources_list_fallback_ok(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_noble"), + str(FAKE_APT_DIRS / "noble"), ): repository_mapping = apt.RepositoryMapping() assert repository_mapping._repository_map @@ -245,7 +209,7 @@ def test_init_with_bad_ubuntu_sources(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_empty_sources"), + str(FAKE_APT_DIRS / "noble-empty-sources"), ): with self.assertRaises(apt.InvalidSourceError): apt.RepositoryMapping() @@ -254,7 +218,7 @@ def test_init_with_third_party_inkscape_source(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_noble_with_inkscape"), + str(FAKE_APT_DIRS / "noble-with-inkscape"), ): repository_mapping = apt.RepositoryMapping() assert repository_mapping._repository_map @@ -263,7 +227,7 @@ def test_load_deb822_ubuntu_sources(self): with patch.object( apt.RepositoryMapping, "_apt_dir", - str(UNIT_TEST_DIR / "data/fake_apt_dir_empty"), + str(FAKE_APT_DIRS / "empty"), ): repository_mapping = apt.RepositoryMapping() assert not repository_mapping._repository_map @@ -310,37 +274,27 @@ def test_init_with_deb822(self): They should be equivalent with the sample data being used. """ + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble"), + ): + repos_deb822 = apt.RepositoryMapping() - def isnt_file(f: str) -> bool: - return False - - def iglob_list(s: str) -> typing.Iterable[str]: - if s.endswith(".list"): - return ["FILENAME"] - return [] - - def iglob_sources(s: str) -> typing.Iterable[str]: - if s.endswith(".sources"): - return ["FILENAME"] - return [] - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_one_line): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_list): - repository_mapping_list = apt.RepositoryMapping() - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_sources): - repository_mapping_sources = apt.RepositoryMapping() + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-in-one-per-line-format"), + ): + repos_one_per_line = apt.RepositoryMapping() - list_keys = sorted(repository_mapping_list._repository_map.keys()) - sources_keys = sorted(repository_mapping_sources._repository_map.keys()) + list_keys = sorted(repos_one_per_line._repository_map.keys()) + sources_keys = sorted(repos_deb822._repository_map.keys()) assert sources_keys == list_keys for list_key, sources_key in zip(list_keys, sources_keys): - list_repo = repository_mapping_list[list_key] - sources_repo = repository_mapping_sources[sources_key] + list_repo = repos_one_per_line[list_key] + sources_repo = repos_deb822[sources_key] assert list_repo.enabled == sources_repo.enabled assert list_repo.repotype == sources_repo.repotype assert list_repo.uri == sources_repo.uri From da057f5aed98ffac2b99f8226cffe6477aa25fb2 Mon Sep 17 00:00:00 2001 From: James Garner Date: Tue, 3 Dec 2024 19:47:44 +1300 Subject: [PATCH 38/50] tests: iterate on deb822 tests --- lib/charms/operator_libs_linux/v0/apt.py | 110 +++++++++++------- .../noble-with-comments-etc/sources.list | 1 - .../sources.list.d/ubuntu.sources | 14 ++- tests/unit/test_deb822.py | 71 ++++++----- 4 files changed, 126 insertions(+), 70 deletions(-) delete mode 100644 tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 4d25805a..f36cc09f 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -924,6 +924,35 @@ class InvalidSourceError(Error): """Exceptions for invalid source entries.""" +class MissingRequiredKeyError(InvalidSourceError): + """Missing a required value in a source file.""" + def __init__( + self, message: str = "", *, file: str, line: Optional[int], key: str + ) -> None: + super().__init__(message, file, line, key) + self.file = file + self.line = line + self.key = key + + +class BadValueError(InvalidSourceError): + """Bad value for an entry in a source file.""" + def __init__( + self, + message: str = "", + *, + file: str, + line: Optional[int], + key: str, + value: str, + ) -> None: + super().__init__(message, file, line, key, value) + self.file = file + self.line = line + self.key = key + self.value = value + + class GPGKeyError(Error): """Exceptions for GPG keys.""" @@ -1243,6 +1272,7 @@ class RepositoryMapping(Mapping[str, DebianRepository]): _sources_subdir = "sources.list.d" _default_list_name = "sources.list" _default_sources_name = "ubuntu.sources" + _last_errors: Iterable[Error] = () def __init__(self): self._repository_map: Dict[str, DebianRepository] = {} @@ -1316,6 +1346,7 @@ def load_deb822(self, filename: str) -> None: self._repository_map[identifier] = repo if errors: + self._last_errors = errors logger.debug( "the following %d error(s) were encountered when reading deb822 sources:\n%s", len(errors), @@ -1657,16 +1688,12 @@ def _deb822_options_to_repos( elif enabled_field == "no": enabled = False else: - raise InvalidSourceError( - ( - "Bad value '{value}' for entry 'Enabled' (line {enabled_line})" - " in file {file}. If 'Enabled' is present it must be one of" - " yes or no (if absent it defaults to yes)." - ).format( - value=enabled_field, - enabled_line=line_numbers.get("Enabled"), - file=filename, - ) + raise BadValueError( + "Must be one of yes or no (default: yes).", + file=filename, + line=line_numbers.get("Enabled"), + key="Enabled", + value=enabled_field, ) # Signed-By gpg_key_file = options.pop("Signed-By", "") @@ -1682,44 +1709,49 @@ def _deb822_options_to_repos( suites = options.pop("Suites").split() except KeyError as e: [key] = e.args - raise InvalidSourceError( - "Missing required entry '{key}' for entry starting on line {line} in {file}.".format( - key=key, - line=min(line_numbers.values()) if line_numbers else None, - file=filename, - ) + raise MissingRequiredKeyError( + key=key, + line=min(line_numbers.values()) if line_numbers else None, + file=filename, ) # Components + # suite can specify an exact path, in which case the components must be omitted and suite must end with a slash (/). + # If suite does not specify an exact path, at least one component must be present. + # https://manpages.ubuntu.com/manpages/noble/man5/sources.list.5.html components: List[str] if len(suites) == 1 and suites[0].endswith("/"): if "Components" in options: - raise InvalidSourceError( - ( - "Since 'Suites' (line {suites_line}) specifies" - " a path relative to 'URIs' (line {uris_line})," - " 'Components' (line {components_line}) must be ommitted" - " (in file {file})." - ).format( - suites_line=line_numbers.get("Suites"), - uris_line=line_numbers.get("URIs"), - components_line=line_numbers.get("Components"), - file=filename, - ) + msg = ( + "Since 'Suites' (line {suites_line}) specifies" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be ommitted." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise BadValueError( + msg, + file=filename, + line=line_numbers.get("Components"), + key="Components", + value=options["Components"], ) components = [] else: if "Components" not in options: - raise InvalidSourceError( - ( - "Since 'Suites' (line {suites_line}) does not specify" - " a path relative to 'URIs' (line {uris_line})," - " 'Components' must be present in this paragraph" - " (in file {file})." - ).format( - suites_line=line_numbers.get("Suites"), - uris_line=line_numbers.get("URIs"), - file=filename, - ) + msg = ( + "Since 'Suites' (line {suites_line}) does not specify" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be present in this paragraph." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise MissingRequiredKeyError( + msg, + file=filename, + line=min(line_numbers.values()) if line_numbers else None, + key="Components", ) components = options.pop("Components").split() repos = tuple( diff --git a/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list deleted file mode 100644 index eb39b946..00000000 --- a/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list +++ /dev/null @@ -1 +0,0 @@ -# Ubuntu sources have moved to /etc/apt/sources.list.d/ubuntu.sources diff --git a/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources index eff19616..87a867eb 100644 --- a/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources +++ b/tests/unit/data/fake-apt-dirs/noble-with-comments-etc/sources.list.d/ubuntu.sources @@ -30,4 +30,16 @@ Components: main # this is an error, can't have components with an exact path Types: deb URIs: http://nz.archive.ubuntu.com/ubuntu/ Suites: noble noble-updates noble-backports -# this is an error, must have at least one component if suites isn't an exact path \ No newline at end of file +# this is an error, must have at least one component if suites isn't an exact path + +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Enabled: no + +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Enabled: bad \ No newline at end of file diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 688634ff..3ee16337 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -242,32 +242,48 @@ def test_load_deb822_ubuntu_sources(self): ] def test_load_deb822_w_comments(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch( - "builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822_with_comments + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-with-comments-etc"), ): - with patch.object(apt.logger, "debug") as debug: - repository_mapping.load_deb822("FILENAME") + repository_mapping = apt.RepositoryMapping() + # TODO: split cases into separate files and test load_deb822 instead + # this will make things a lot more understandable and maintainable + assert sorted(repository_mapping._repository_map.keys()) == [ + "deb-http://archive.ubuntu.com/ubuntu/-noble", + "deb-http://archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://archive.ubuntu.com/ubuntu/-noble-updates", + "deb-http://nz.archive.ubuntu.com/ubuntu/-an/exact/path/", "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + "deb-src-http://archive.ubuntu.com/ubuntu/-noble", + "deb-src-http://archive.ubuntu.com/ubuntu/-noble-backports", + "deb-src-http://archive.ubuntu.com/ubuntu/-noble-updates", + "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", ] - debug.assert_called_once_with( - ANY, - 1, # number of errors - "Missing required entry 'Types' for entry starting on line 12 in FILENAME.", - ) + errors = tuple(repository_mapping._last_errors) + assert len(errors) == 4 + ( + missing_types, + components_not_ommitted, + components_not_present, + bad_enabled_value, + ) = errors + assert isinstance(missing_types, apt.MissingRequiredKeyError) + assert missing_types.key == "Types" + assert isinstance(components_not_ommitted, apt.BadValueError) + assert components_not_ommitted.key == "Components" + assert components_not_ommitted.value == "main" + assert isinstance(components_not_present, apt.MissingRequiredKeyError) + assert components_not_present.key == "Components" + assert isinstance(bad_enabled_value, apt.BadValueError) + assert bad_enabled_value.key == "Enabled" + assert bad_enabled_value.value == "bad" def test_init_with_deb822(self): """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. @@ -307,15 +323,12 @@ def test_init_with_deb822(self): ) def test_disable_with_deb822(self): - def isnt_file(f: str) -> bool: - return False - - def iglob_nothing(s: str) -> typing.Iterable[str]: - return [] - - with patch("os.path.isfile", new=isnt_file): - with patch("glob.iglob", new=iglob_nothing): - repository_mapping = apt.RepositoryMapping() + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "empty"), + ): + repository_mapping = apt.RepositoryMapping() repo = apt.DebianRepository( enabled=True, repotype="deb", From 08aa90d74c71cb9396a3641483257c6330c134fe Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 12:46:17 +1300 Subject: [PATCH 39/50] tests: fully refactor and expand deb822 unit tests --- lib/charms/operator_libs_linux/v0/apt.py | 12 +- ...ponents-missing-without-exact-path.sources | 4 + ...components-present-with-exact-path.sources | 4 + .../bad-stanza-enabled-bad.sources | 5 + .../bad-stanza-missing-required-keys.sources | 3 + .../good-stanza-comments.sources | 9 + .../good-stanza-enabled-no.sources | 5 + .../good-stanza-exact-path.sources | 3 + .../good-stanza-noble-main-etc.sources | 5 + .../good-stanza-noble-security.sources | 5 + .../good-stanza-with-gpg-key.sources | 34 + .../stanzas-fully-commented-out.sources | 13 + .../individual-files/stanzas-noble.sources | 11 + .../stanzas-one-good-one-bad-comments.sources | 15 + tests/unit/test_deb822.py | 660 +++++++++--------- 15 files changed, 468 insertions(+), 320 deletions(-) create mode 100644 tests/unit/data/individual-files/bad-stanza-components-missing-without-exact-path.sources create mode 100644 tests/unit/data/individual-files/bad-stanza-components-present-with-exact-path.sources create mode 100644 tests/unit/data/individual-files/bad-stanza-enabled-bad.sources create mode 100644 tests/unit/data/individual-files/bad-stanza-missing-required-keys.sources create mode 100644 tests/unit/data/individual-files/good-stanza-comments.sources create mode 100644 tests/unit/data/individual-files/good-stanza-enabled-no.sources create mode 100644 tests/unit/data/individual-files/good-stanza-exact-path.sources create mode 100644 tests/unit/data/individual-files/good-stanza-noble-main-etc.sources create mode 100644 tests/unit/data/individual-files/good-stanza-noble-security.sources create mode 100644 tests/unit/data/individual-files/good-stanza-with-gpg-key.sources create mode 100644 tests/unit/data/individual-files/stanzas-fully-commented-out.sources create mode 100644 tests/unit/data/individual-files/stanzas-noble.sources create mode 100644 tests/unit/data/individual-files/stanzas-one-good-one-bad-comments.sources diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index f36cc09f..a58dc44c 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -127,6 +127,7 @@ VALID_SOURCE_TYPES = ("deb", "deb-src") OPTIONS_MATCHER = re.compile(r"\[.*?\]") +_GPG_KEY_DIR = "/etc/apt/trusted.gpg.d/" class Error(Exception): @@ -894,7 +895,7 @@ def import_key(key: str) -> str: key_bytes = key.encode("utf-8") key_name = DebianRepository._get_keyid_by_gpg_key(key_bytes) key_gpg = DebianRepository._dearmor_gpg_key(key_bytes) - gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key_name) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, "{}.gpg".format(key_name)) DebianRepository._write_apt_gpg_keyfile( key_name=gpg_key_filename, key_material=key_gpg ) @@ -915,7 +916,7 @@ def import_key(key: str) -> str: key_asc = DebianRepository._get_key_by_keyid(key) # write the key in GPG format so that apt-key list shows it key_gpg = DebianRepository._dearmor_gpg_key(key_asc.encode("utf-8")) - gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, "{}.gpg".format(key)) DebianRepository._write_apt_gpg_keyfile(key_name=gpg_key_filename, key_material=key_gpg) return gpg_key_filename @@ -1272,7 +1273,7 @@ class RepositoryMapping(Mapping[str, DebianRepository]): _sources_subdir = "sources.list.d" _default_list_name = "sources.list" _default_sources_name = "ubuntu.sources" - _last_errors: Iterable[Error] = () + _last_errors: Tuple[Error, ...] = () def __init__(self): self._repository_map: Dict[str, DebianRepository] = {} @@ -1537,7 +1538,7 @@ def _add_apt_repository( raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) line = repo._to_line() # pyright: ignore[reportPrivateUsage] cmd = [ - "apt-add-repository", + "add-apt-repository", "--yes", "--sourceslist=" + line, ] @@ -1597,9 +1598,10 @@ def repositories(self) -> Tuple[DebianRepository, ...]: return self._repositories def get_gpg_key_filename(self) -> str: - """Return the path to the GPG key for this repository. + """Return the path to the GPG key for this stanza. Import the key first, if the key itself was provided in the stanza. + Return an empty string if no filename or key was provided. """ if self._gpg_key_filename: return self._gpg_key_filename diff --git a/tests/unit/data/individual-files/bad-stanza-components-missing-without-exact-path.sources b/tests/unit/data/individual-files/bad-stanza-components-missing-without-exact-path.sources new file mode 100644 index 00000000..a8d58999 --- /dev/null +++ b/tests/unit/data/individual-files/bad-stanza-components-missing-without-exact-path.sources @@ -0,0 +1,4 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +# this is an error, must have at least one component if suites isn't an exact path \ No newline at end of file diff --git a/tests/unit/data/individual-files/bad-stanza-components-present-with-exact-path.sources b/tests/unit/data/individual-files/bad-stanza-components-present-with-exact-path.sources new file mode 100644 index 00000000..b0cd1ec7 --- /dev/null +++ b/tests/unit/data/individual-files/bad-stanza-components-present-with-exact-path.sources @@ -0,0 +1,4 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: an/exact/path/ +Components: main # this is an error, can't have components with an exact path \ No newline at end of file diff --git a/tests/unit/data/individual-files/bad-stanza-enabled-bad.sources b/tests/unit/data/individual-files/bad-stanza-enabled-bad.sources new file mode 100644 index 00000000..2d707b53 --- /dev/null +++ b/tests/unit/data/individual-files/bad-stanza-enabled-bad.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Enabled: bad \ No newline at end of file diff --git a/tests/unit/data/individual-files/bad-stanza-missing-required-keys.sources b/tests/unit/data/individual-files/bad-stanza-missing-required-keys.sources new file mode 100644 index 00000000..6eca1805 --- /dev/null +++ b/tests/unit/data/individual-files/bad-stanza-missing-required-keys.sources @@ -0,0 +1,3 @@ +# a comment + +Foo: Bar # this is a separate (malformed) entry \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-comments.sources b/tests/unit/data/individual-files/good-stanza-comments.sources new file mode 100644 index 00000000..1f4972ae --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-comments.sources @@ -0,0 +1,9 @@ +Components: main restricted universe multiverse # Components first! I don't think deb822 cares about ordering +Types: deb deb-src # this could be one or both of these options +URIs: http://nz.archive.ubuntu.com/ubuntu/ http://archive.ubuntu.com/ubuntu/ + # there can be multiple space separated URIs + # sources are specified in priority order + # apt does some de-duplication of sources after parsing too +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +# let's make this insecure! (jk, just testing parsing) +Suites: noble noble-updates noble-backports \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-enabled-no.sources b/tests/unit/data/individual-files/good-stanza-enabled-no.sources new file mode 100644 index 00000000..9c48d4e2 --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-enabled-no.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Enabled: no \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-exact-path.sources b/tests/unit/data/individual-files/good-stanza-exact-path.sources new file mode 100644 index 00000000..6b2a587a --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-exact-path.sources @@ -0,0 +1,3 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: an/exact/path/ \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-noble-main-etc.sources b/tests/unit/data/individual-files/good-stanza-noble-main-etc.sources new file mode 100644 index 00000000..777a4f4c --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-noble-main-etc.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-noble-security.sources b/tests/unit/data/individual-files/good-stanza-noble-security.sources new file mode 100644 index 00000000..c1dcc762 --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-noble-security.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg \ No newline at end of file diff --git a/tests/unit/data/individual-files/good-stanza-with-gpg-key.sources b/tests/unit/data/individual-files/good-stanza-with-gpg-key.sources new file mode 100644 index 00000000..9bce7f16 --- /dev/null +++ b/tests/unit/data/individual-files/good-stanza-with-gpg-key.sources @@ -0,0 +1,34 @@ +Types: deb +URIs: https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/ +Suites: noble +Components: main +Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU + 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM + 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 + vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U + NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu + 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP + pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww + 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e + ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA + RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ + S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB + tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 + FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC + AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle + amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO + qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu + oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R + PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE + MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom + 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 + J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ + q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z + lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k + uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO + 2ekz6IVTXVA= + =VF33 + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/data/individual-files/stanzas-fully-commented-out.sources b/tests/unit/data/individual-files/stanzas-fully-commented-out.sources new file mode 100644 index 00000000..791814d7 --- /dev/null +++ b/tests/unit/data/individual-files/stanzas-fully-commented-out.sources @@ -0,0 +1,13 @@ +#a fully commented out stanza +#Types: deb # with an inline comment too +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +#another fully commented out stanza +#Types: deb-src # with an inline comment too +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg \ No newline at end of file diff --git a/tests/unit/data/individual-files/stanzas-noble.sources b/tests/unit/data/individual-files/stanzas-noble.sources new file mode 100644 index 00000000..021ad293 --- /dev/null +++ b/tests/unit/data/individual-files/stanzas-noble.sources @@ -0,0 +1,11 @@ +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +Types: deb +URIs: http://security.ubuntu.com/ubuntu +Suites: noble-security +Components: main restricted universe multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg \ No newline at end of file diff --git a/tests/unit/data/individual-files/stanzas-one-good-one-bad-comments.sources b/tests/unit/data/individual-files/stanzas-one-good-one-bad-comments.sources new file mode 100644 index 00000000..0b2195a5 --- /dev/null +++ b/tests/unit/data/individual-files/stanzas-one-good-one-bad-comments.sources @@ -0,0 +1,15 @@ +# a good entry that defines one repository +Types: deb +URIs: http://nz.archive.ubuntu.com/ubuntu/ +Suites: noble +Components: main + +Foo: Bar # this is a separate (malformed) entry + +# this is fully commented out and will be skipped entirely +#Types: deb +#URIs: http://security.ubuntu.com/ubuntu +#Suites: noble-security +#Components: main restricted universe multiverse +#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +## disable security updates while we're at it \ No newline at end of file diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 3ee16337..1a2a2a8e 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -3,339 +3,369 @@ # pyright: reportPrivateUsage=false -import typing -import unittest +import itertools +import tempfile from pathlib import Path -from unittest.mock import ANY, mock_open, patch +from unittest.mock import patch +import pytest from charms.operator_libs_linux.v0 import apt TEST_DATA_DIR = Path(__file__).parent / "data" FAKE_APT_DIRS = TEST_DATA_DIR / "fake-apt-dirs" +SOURCES_DIR = TEST_DATA_DIR / "individual-files" + + +@pytest.fixture +def repo_mapping(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "empty"), + ): + repository_mapping = apt.RepositoryMapping() + return repository_mapping + + +def test_init_no_files(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "empty"), + ): + repository_mapping = apt.RepositoryMapping() + assert not repository_mapping._repository_map + + +def test_init_with_good_sources_list(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "bionic"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + +def test_init_with_bad_sources_list_no_fallback(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-no-sources"), + ): + with pytest.raises(apt.InvalidSourceError): + apt.RepositoryMapping() + + +def test_init_with_bad_sources_list_fallback_ok(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + +def test_init_with_bad_ubuntu_sources(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-empty-sources"), + ): + with pytest.raises(apt.InvalidSourceError): + apt.RepositoryMapping() + + +def test_init_with_third_party_inkscape_source(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-with-inkscape"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + +def test_init_w_comments(): + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-with-comments-etc"), + ): + repository_mapping = apt.RepositoryMapping() + assert repository_mapping._repository_map + + +def test_deb822_format_equivalence(): + """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. + + They should be equivalent with the sample data being used. + """ + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble"), + ): + repos_deb822 = apt.RepositoryMapping() + + with patch.object( + apt.RepositoryMapping, + "_apt_dir", + str(FAKE_APT_DIRS / "noble-in-one-per-line-format"), + ): + repos_one_per_line = apt.RepositoryMapping() + + list_keys = sorted(repos_one_per_line._repository_map.keys()) + sources_keys = sorted(repos_deb822._repository_map.keys()) + assert sources_keys == list_keys + + for list_key, sources_key in zip(list_keys, sources_keys): + list_repo = repos_one_per_line[list_key] + sources_repo = repos_deb822[sources_key] + assert list_repo.enabled == sources_repo.enabled + assert list_repo.repotype == sources_repo.repotype + assert list_repo.uri == sources_repo.uri + assert list_repo.release == sources_repo.release + assert list_repo.groups == sources_repo.groups + assert list_repo.gpg_key == sources_repo.gpg_key + assert ( + list_repo.options # pyright: ignore[reportUnknownMemberType] + == sources_repo.options # pyright: ignore[reportUnknownMemberType] + ) -ubuntu_sources_deb822 = """ -Types: deb -URIs: http://nz.archive.ubuntu.com/ubuntu/ -Suites: noble noble-updates noble-backports -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg - -Types: deb -URIs: http://security.ubuntu.com/ubuntu -Suites: noble-security -Components: main restricted universe multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -""" - -ubuntu_sources_deb822_with_comments = """ -Components: main restricted universe multiverse # this lib doesn't care about order -Types: deb # this could include deb-src as well or instead -URIs: http://nz.archive.ubuntu.com/ubuntu/ - # there can be multiple space separated URIs - # sources are specified in priority order - # apt does some de-duplication of sources after parsing too -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -# let's make this insecure! (jk, just testing parsing) -Suites: noble noble-updates noble-backports - -Foo: Bar # this is a separate (malformed) entry - -#Types: deb -#URIs: http://security.ubuntu.com/ubuntu -#Suites: noble-security -#Components: main restricted universe multiverse -#Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -## disable security updates while we're at it -""" - -ubuntu_sources_one_line = """ -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-updates main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://nz.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse -deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://security.ubuntu.com/ubuntu noble-security main restricted universe multiverse -""" - - -class TestRepositoryMappingDeb822Behaviour(unittest.TestCase): - def test_iter_deb822_paragraphs_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - stanzas = list(apt._iter_deb822_stanzas(lines)) - assert len(stanzas) == 2 - stanza_1, stanza_2 = stanzas - assert len(stanza_1) == 5 - assert len(stanza_2) == 5 - line_numbers = [n for stanza in stanzas for n, _line in stanza] - assert len(set(line_numbers)) == len(line_numbers) # unique line numbers - - def test_iter_deb822_paragraphs_ubuntu_sources_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - stanzas = list(apt._iter_deb822_stanzas(lines)) - assert len(stanzas) == 2 - stanza_1, stanza_2 = stanzas - assert len(stanza_1) == 4 - assert len(stanza_2) == 1 - line_numbers = [n for stanza in stanzas for n, _line in stanza] - assert len(set(line_numbers)) == len(line_numbers) # unique line numbers - - def test_get_deb822_options_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - opts = [apt._deb822_stanza_to_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, _opts_0_line_numbers = opts_0 - opts_1_options, _opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Components": "main restricted universe multiverse", - "Suites": "noble noble-updates noble-backports", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - assert opts_1_options == { - "Types": "deb", - "URIs": "http://security.ubuntu.com/ubuntu", - "Components": "main restricted universe multiverse", - "Suites": "noble-security", - "Signed-By": "/usr/share/keyrings/ubuntu-archive-keyring.gpg", - } - - def test_get_deb822_options_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - paras = list(apt._iter_deb822_stanzas(lines)) - opts = [apt._deb822_stanza_to_options(p) for p in paras] - opts_0, opts_1 = opts - opts_0_options, _opts_0_line_numbers = opts_0 - opts_1_options, _opts_1_line_numbers = opts_1 - assert opts_0_options == { - "Components": "main restricted universe multiverse", - "Types": "deb", - "URIs": "http://nz.archive.ubuntu.com/ubuntu/", - "Suites": "noble noble-updates noble-backports", - } - assert opts_1_options == {"Foo": "Bar"} - - def test_parse_deb822_paragraph_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - main, security = apt._iter_deb822_stanzas(lines) - repos = apt._Deb822Stanza(main).repositories - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - repos = apt._Deb822Stanza(security).repositories - assert len(repos) == 1 - [repo] = repos - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://security.ubuntu.com/ubuntu" - assert repo.release == "noble-security" - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" +def test_load_deb822_missing_components(repo_mapping: apt.RepositoryMapping): + with pytest.raises(apt.InvalidSourceError): + repo_mapping.load_deb822( + str(SOURCES_DIR / "bad-stanza-components-missing-without-exact-path.sources") + ) + assert len(repo_mapping._last_errors) == 1 + [error] = repo_mapping._last_errors + assert isinstance(error, apt.MissingRequiredKeyError) + assert error.key == "Components" - def test_parse_deb822_paragraph_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - ok_stanza, bad_stanza = apt._iter_deb822_stanzas(lines) - repos = apt._Deb822Stanza(ok_stanza).repositories - assert len(repos) == 3 - for repo, suite in zip(repos, ("noble", "noble-updates", "noble-backports")): - assert repo.enabled - assert repo.repotype == "deb" - assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" - assert repo.release == suite - assert repo.groups == ["main", "restricted", "universe", "multiverse"] - assert repo.filename == "" - assert repo.gpg_key == "" - with self.assertRaises(apt.InvalidSourceError): - apt._Deb822Stanza(bad_stanza) - - def test_parse_deb822_lines_ubuntu_sources(self): - lines = ubuntu_sources_deb822.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - assert len(repos) == 4 - assert not errors - - def test_parse_deb822_lines_w_comments(self): - lines = ubuntu_sources_deb822_with_comments.strip().split("\n") - repos, errors = apt.RepositoryMapping._parse_deb822_lines(lines) - assert len(repos) == 3 - assert len(errors) == 1 - [error] = errors - assert isinstance(error, apt.InvalidSourceError) - - def test_init_no_files(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "empty"), - ): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - def test_init_with_good_sources_list(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "bionic"), - ): - repository_mapping = apt.RepositoryMapping() - assert repository_mapping._repository_map +def test_load_deb822_components_with_exact_path(repo_mapping: apt.RepositoryMapping): + with pytest.raises(apt.InvalidSourceError): + repo_mapping.load_deb822( + str(SOURCES_DIR / "bad-stanza-components-present-with-exact-path.sources") + ) + assert len(repo_mapping._last_errors) == 1 + [error] = repo_mapping._last_errors + assert isinstance(error, apt.BadValueError) + assert error.key == "Components" - def test_init_with_bad_sources_list_no_fallback(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble-no-sources"), - ): - with self.assertRaises(apt.InvalidSourceError): - apt.RepositoryMapping() - def test_init_with_bad_sources_list_fallback_ok(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble"), - ): - repository_mapping = apt.RepositoryMapping() - assert repository_mapping._repository_map +def test_load_deb822_bad_enabled_value(repo_mapping: apt.RepositoryMapping): + with pytest.raises(apt.InvalidSourceError): + repo_mapping.load_deb822( + str(SOURCES_DIR / "bad-stanza-enabled-bad.sources") + ) + assert len(repo_mapping._last_errors) == 1 + [error] = repo_mapping._last_errors + assert isinstance(error, apt.BadValueError) + assert error.key == "Enabled" - def test_init_with_bad_ubuntu_sources(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble-empty-sources"), - ): - with self.assertRaises(apt.InvalidSourceError): - apt.RepositoryMapping() - def test_init_with_third_party_inkscape_source(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble-with-inkscape"), - ): - repository_mapping = apt.RepositoryMapping() - assert repository_mapping._repository_map +def test_load_deb822_missing_required_keys(repo_mapping: apt.RepositoryMapping): + with pytest.raises(apt.InvalidSourceError): + repo_mapping.load_deb822( + str(SOURCES_DIR / "bad-stanza-missing-required-keys.sources") + ) + assert len(repo_mapping._last_errors) == 1 + [error] = repo_mapping._last_errors + assert isinstance(error, apt.MissingRequiredKeyError) + assert error.key in ("Types", "URIs", "Suites") + + +def test_load_deb822_comments(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-comments.sources") + repo_mapping.load_deb822(filename) + repotypes = ("deb", "deb-src") + uris = ("http://nz.archive.ubuntu.com/ubuntu/", "http://archive.ubuntu.com/ubuntu/") + suites = ("noble", "noble-updates", "noble-backports") + assert len(repo_mapping) == len(repotypes) * len(uris) * len(suites) + for repo, (repotype, uri, suite) in zip( + repo_mapping, itertools.product(repotypes, uris, suites) + ): + assert isinstance(repo, apt.DebianRepository) + assert repo.enabled + assert repo.repotype == repotype + assert repo.uri == uri + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == filename + assert repo.gpg_key == "" + + +def test_load_deb822_enabled_no(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-enabled-no.sources") + repo_mapping.load_deb822(filename) + for repo in repo_mapping: + assert isinstance(repo, apt.DebianRepository) + assert not repo.enabled + + +def test_load_deb822_exact_path(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-exact-path.sources") + repo_mapping.load_deb822(filename) + [repo] = repo_mapping + assert isinstance(repo, apt.DebianRepository) + assert repo.uri + assert "/" in repo.release + assert not repo.groups + + +def test_load_deb822_fully_commented_out_stanzas(repo_mapping: apt.RepositoryMapping): + with pytest.raises(apt.InvalidSourceError): + repo_mapping.load_deb822(str(SOURCES_DIR / "stanzas-fully-commented-out.sources")) + assert not repo_mapping._repository_map + assert not repo_mapping._last_errors # no individual errors, just no good entries + + +def test_load_deb822_one_good_stanza_one_bad(repo_mapping: apt.RepositoryMapping): + repo_mapping.load_deb822(str(SOURCES_DIR / "stanzas-one-good-one-bad-comments.sources")) + repos = repo_mapping._repository_map.values() + errors = repo_mapping._last_errors + assert len(repos) == 1 # one good stanza defines one repository + assert len(errors) == 1 # one stanza was bad + [error] = errors + assert isinstance(error, apt.MissingRequiredKeyError) + + +def test_load_deb822_ubuntu_sources(repo_mapping: apt.RepositoryMapping): + assert not repo_mapping._repository_map + repo_mapping.load_deb822(str(SOURCES_DIR / "stanzas-noble.sources")) + assert sorted(repo_mapping._repository_map.keys()) == [ + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", + "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", + "deb-http://security.ubuntu.com/ubuntu-noble-security", + ] + assert not repo_mapping._last_errors + + +def test_load_deb822_with_gpg_key(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-with-gpg-key.sources") + repo_mapping.load_deb822(filename) + [repo] = repo_mapping + assert isinstance(repo, apt.DebianRepository) + # DebianRepository.gpg_key is expected to be a string file path + # the inkscape sources file provides the key inline + # in this case the library imports the key to a file on first access + with tempfile.TemporaryDirectory() as tmpdir: + assert not list(Path(tmpdir).iterdir()) + with patch.object(apt, "_GPG_KEY_DIR", tmpdir): + inkscape_key_file = repo.gpg_key + key_paths = list(Path(tmpdir).iterdir()) + assert len(key_paths) == 1 + [key_path] = key_paths + assert Path(inkscape_key_file).name == key_path.name + assert Path(inkscape_key_file).parent == key_path.parent + # the filename is cached for subsequent access + with tempfile.TemporaryDirectory() as tmpdir: + assert not list(Path(tmpdir).iterdir()) + with patch.object(apt, "_GPG_KEY_DIR", tmpdir): + inkscape_key_file_cached = repo.gpg_key + assert not list(Path(tmpdir).iterdir()) + assert inkscape_key_file == inkscape_key_file_cached + + +def test_load_deb822_stanza_ubuntu_main_etc(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-noble-main-etc.sources") + repo_mapping.load_deb822(filename) + assert len(repo_mapping) == 3 + for repo, suite in zip(repo_mapping, ("noble", "noble-updates", "noble-backports")): + assert isinstance(repo, apt.DebianRepository) + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://nz.archive.ubuntu.com/ubuntu/" + assert repo.release == suite + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == filename + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" - def test_load_deb822_ubuntu_sources(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "empty"), - ): - repository_mapping = apt.RepositoryMapping() - assert not repository_mapping._repository_map - - with patch("builtins.open", new_callable=mock_open, read_data=ubuntu_sources_deb822): - repository_mapping.load_deb822("") - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - "deb-http://security.ubuntu.com/ubuntu-noble-security", - ] - - def test_load_deb822_w_comments(self): - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble-with-comments-etc"), - ): - repository_mapping = apt.RepositoryMapping() - # TODO: split cases into separate files and test load_deb822 instead - # this will make things a lot more understandable and maintainable - - assert sorted(repository_mapping._repository_map.keys()) == [ - "deb-http://archive.ubuntu.com/ubuntu/-noble", - "deb-http://archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://archive.ubuntu.com/ubuntu/-noble-updates", - "deb-http://nz.archive.ubuntu.com/ubuntu/-an/exact/path/", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - "deb-src-http://archive.ubuntu.com/ubuntu/-noble", - "deb-src-http://archive.ubuntu.com/ubuntu/-noble-backports", - "deb-src-http://archive.ubuntu.com/ubuntu/-noble-updates", - "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble", - "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble-backports", - "deb-src-http://nz.archive.ubuntu.com/ubuntu/-noble-updates", - ] - errors = tuple(repository_mapping._last_errors) - assert len(errors) == 4 - ( - missing_types, - components_not_ommitted, - components_not_present, - bad_enabled_value, - ) = errors - assert isinstance(missing_types, apt.MissingRequiredKeyError) - assert missing_types.key == "Types" - assert isinstance(components_not_ommitted, apt.BadValueError) - assert components_not_ommitted.key == "Components" - assert components_not_ommitted.value == "main" - assert isinstance(components_not_present, apt.MissingRequiredKeyError) - assert components_not_present.key == "Components" - assert isinstance(bad_enabled_value, apt.BadValueError) - assert bad_enabled_value.key == "Enabled" - assert bad_enabled_value.value == "bad" - - def test_init_with_deb822(self): - """Mock file opening to initialise a RepositoryMapping from deb822 and one-line-style. - - They should be equivalent with the sample data being used. - """ - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble"), - ): - repos_deb822 = apt.RepositoryMapping() - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "noble-in-one-per-line-format"), - ): - repos_one_per_line = apt.RepositoryMapping() - - list_keys = sorted(repos_one_per_line._repository_map.keys()) - sources_keys = sorted(repos_deb822._repository_map.keys()) - assert sources_keys == list_keys - - for list_key, sources_key in zip(list_keys, sources_keys): - list_repo = repos_one_per_line[list_key] - sources_repo = repos_deb822[sources_key] - assert list_repo.enabled == sources_repo.enabled - assert list_repo.repotype == sources_repo.repotype - assert list_repo.uri == sources_repo.uri - assert list_repo.release == sources_repo.release - assert list_repo.groups == sources_repo.groups - assert list_repo.gpg_key == sources_repo.gpg_key - assert ( - list_repo.options # pyright: ignore[reportUnknownMemberType] - == sources_repo.options # pyright: ignore[reportUnknownMemberType] - ) - - def test_disable_with_deb822(self): +def test_load_deb822_stanza_ubuntu_security(repo_mapping: apt.RepositoryMapping): + filename = str(SOURCES_DIR / "good-stanza-noble-security.sources") + repo_mapping.load_deb822(filename) + assert len(repo_mapping) == 1 + [repo] = repo_mapping + assert isinstance(repo, apt.DebianRepository) + assert repo.enabled + assert repo.repotype == "deb" + assert repo.uri == "http://security.ubuntu.com/ubuntu" + assert repo.release == "noble-security" + assert repo.groups == ["main", "restricted", "universe", "multiverse"] + assert repo.filename == filename + assert repo.gpg_key == "/usr/share/keyrings/ubuntu-archive-keyring.gpg" + + +def test_disable_with_deb822(repo_mapping: apt.RepositoryMapping): + repo = apt.DebianRepository( + enabled=True, + repotype="deb", + uri="http://nz.archive.ubuntu.com/ubuntu/", + release="noble", + groups=["main", "restricted"], + ) + repo._deb822_stanza = apt._Deb822Stanza(numbered_lines=[]) + with pytest.raises(NotImplementedError): + repo_mapping.disable(repo) + + +def test_add_with_deb822(repo_mapping: apt.RepositoryMapping): + with (SOURCES_DIR / "good-stanza-exact-path.sources").open() as f: + repos, errors = repo_mapping._parse_deb822_lines(f) + assert len(repos) == 1 + assert not errors + [repo] = repos + identifier = repo._get_identifier() + # call add with update_cache=True + with patch.object(apt.subprocess, "run") as mock_run_1: with patch.object( apt.RepositoryMapping, "_apt_dir", str(FAKE_APT_DIRS / "empty"), ): - repository_mapping = apt.RepositoryMapping() - repo = apt.DebianRepository( - enabled=True, - repotype="deb", - uri="http://nz.archive.ubuntu.com/ubuntu/", - release="noble", - groups=["main", "restricted"], - ) - repo._deb822_stanza = apt._Deb822Stanza(numbered_lines=[]) - with self.assertRaises(NotImplementedError): - repository_mapping.disable(repo) + r1 = repo_mapping.add(repo, update_cache=True) + assert r1 is not repo_mapping + assert identifier not in repo_mapping # because it's the old mapping + assert identifier not in r1 # because we mocked out the subprocess call + mock_run_1.assert_called_once_with( + [ + "add-apt-repository", + "--yes", + "--sourceslist=deb http://nz.archive.ubuntu.com/ubuntu/ an/exact/path/ ", + ], + check=True, + capture_output=True, + ) + # call add with update_cache=False + with patch.object(apt.subprocess, "run") as mock_run_2: + r2 = repo_mapping.add(repo) # update_cache=False by default + assert r2 is repo_mapping + assert identifier in repo_mapping + mock_run_2.assert_called_once_with( + [ + "add-apt-repository", + "--yes", + "--sourceslist=deb http://nz.archive.ubuntu.com/ubuntu/ an/exact/path/ ", + "--no-update", + ], + check=True, + capture_output=True, + ) + # we re-raise CalledProcessError after logging + error = apt.CalledProcessError(1, 'cmd') + error.stdout = error.stderr = b'' + with patch.object(apt.logger, "error") as mock_logging_error: + with patch.object(apt.subprocess, "run", side_effect=error): + with pytest.raises(apt.CalledProcessError): + repo_mapping.add(repo) + mock_logging_error.assert_called_once() + # call add with a disabled repository + repo._enabled = False + with pytest.raises(ValueError): + repo_mapping.add(repo) From 8008a971a98a31efa1811952c33706c486525aa0 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 13:16:18 +1300 Subject: [PATCH 40/50] style: make linting happy --- lib/charms/operator_libs_linux/v0/apt.py | 6 +++--- tests/unit/test_deb822.py | 12 ++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index a58dc44c..78bfbe33 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -927,9 +927,8 @@ class InvalidSourceError(Error): class MissingRequiredKeyError(InvalidSourceError): """Missing a required value in a source file.""" - def __init__( - self, message: str = "", *, file: str, line: Optional[int], key: str - ) -> None: + + def __init__(self, message: str = "", *, file: str, line: Optional[int], key: str) -> None: super().__init__(message, file, line, key) self.file = file self.line = line @@ -938,6 +937,7 @@ def __init__( class BadValueError(InvalidSourceError): """Bad value for an entry in a source file.""" + def __init__( self, message: str = "", diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 1a2a2a8e..0bad2df5 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -159,9 +159,7 @@ def test_load_deb822_components_with_exact_path(repo_mapping: apt.RepositoryMapp def test_load_deb822_bad_enabled_value(repo_mapping: apt.RepositoryMapping): with pytest.raises(apt.InvalidSourceError): - repo_mapping.load_deb822( - str(SOURCES_DIR / "bad-stanza-enabled-bad.sources") - ) + repo_mapping.load_deb822(str(SOURCES_DIR / "bad-stanza-enabled-bad.sources")) assert len(repo_mapping._last_errors) == 1 [error] = repo_mapping._last_errors assert isinstance(error, apt.BadValueError) @@ -170,9 +168,7 @@ def test_load_deb822_bad_enabled_value(repo_mapping: apt.RepositoryMapping): def test_load_deb822_missing_required_keys(repo_mapping: apt.RepositoryMapping): with pytest.raises(apt.InvalidSourceError): - repo_mapping.load_deb822( - str(SOURCES_DIR / "bad-stanza-missing-required-keys.sources") - ) + repo_mapping.load_deb822(str(SOURCES_DIR / "bad-stanza-missing-required-keys.sources")) assert len(repo_mapping._last_errors) == 1 [error] = repo_mapping._last_errors assert isinstance(error, apt.MissingRequiredKeyError) @@ -358,8 +354,8 @@ def test_add_with_deb822(repo_mapping: apt.RepositoryMapping): capture_output=True, ) # we re-raise CalledProcessError after logging - error = apt.CalledProcessError(1, 'cmd') - error.stdout = error.stderr = b'' + error = apt.CalledProcessError(1, "cmd") + error.stdout = error.stderr = b"" with patch.object(apt.logger, "error") as mock_logging_error: with patch.object(apt.subprocess, "run", side_effect=error): with pytest.raises(apt.CalledProcessError): From 58c61bef9d2cfb4b15952285cedbbd53a065b28c Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 13:24:04 +1300 Subject: [PATCH 41/50] test: fix an erroneous assert in integration tests --- tests/integration/test_apt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index 8baa0518..ea91ea62 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -132,7 +132,8 @@ def test_install_package_from_external_repository(): repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] assert repo_id not in repos_after repos_before_update = apt.RepositoryMapping() - assert repo_id in repos_before_update # update hasn't been called yet! + assert repo_id not in repos_before_update + # ^ although apt update hasn't been called yet, we've already modified files on disk apt.update() repos_clean = apt.RepositoryMapping() assert repo_id not in repos_clean From 89014aa238fbd1be2bcd26a4a1a446d0c649b7a8 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 15:18:24 +1300 Subject: [PATCH 42/50] refactor: clean up diff, signatures, etc --- lib/charms/operator_libs_linux/v0/apt.py | 286 ++++++++++------------- tests/integration/test_apt.py | 162 ++++++------- tests/unit/test_deb822.py | 30 +-- tests/unit/test_repo.py | 9 +- 4 files changed, 200 insertions(+), 287 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 78bfbe33..6d14514f 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -102,7 +102,6 @@ import fileinput import glob -import itertools import logging import os import re @@ -807,22 +806,17 @@ def _add( def remove_package( - package_names: Union[str, List[str]], - autoremove: bool = False, + package_names: Union[str, List[str]] ) -> Union[DebianPackage, List[DebianPackage]]: """Remove package(s) from the system. Args: package_names: the name of a package - autoremove: run `apt autoremove` after uninstalling packages. - You probably want to do this if you're removing a metapackage, - otherwise the concrete packages will still be there. - False by default for backwards compatibility. Raises: - PackageNotFoundError if the package is not found. + TypeError: if no packages are provided """ - packages = [] + packages: List[DebianPackage] = [] package_names = [package_names] if isinstance(package_names, str) else package_names if not package_names: @@ -836,9 +830,6 @@ def remove_package( except PackageNotFoundError: logger.info("package '%s' was requested for removal, but it was not installed.", p) - if autoremove: - subprocess.run(["apt", "autoremove"], check=True) - # the list of packages will be empty when no package is removed logger.debug("packages: '%s'", packages) return packages[0] if len(packages) == 1 else packages @@ -925,35 +916,6 @@ class InvalidSourceError(Error): """Exceptions for invalid source entries.""" -class MissingRequiredKeyError(InvalidSourceError): - """Missing a required value in a source file.""" - - def __init__(self, message: str = "", *, file: str, line: Optional[int], key: str) -> None: - super().__init__(message, file, line, key) - self.file = file - self.line = line - self.key = key - - -class BadValueError(InvalidSourceError): - """Bad value for an entry in a source file.""" - - def __init__( - self, - message: str = "", - *, - file: str, - line: Optional[int], - key: str, - value: str, - ) -> None: - super().__init__(message, file, line, key, value) - self.file = file - self.line = line - self.key = key - self.value = value - - class GPGKeyError(Error): """Exceptions for GPG keys.""" @@ -984,24 +946,6 @@ def __init__( self._gpg_key_filename = gpg_key_filename self._options = options - def _get_identifier(self) -> str: - """Return str identifier derived from repotype, uri, and release. - - Private method used to produce the identifiers used by RepositoryMapping. - """ - return "{}-{}-{}".format(self.repotype, self.uri, self.release) - - def _to_line(self) -> str: - """Return the one-per-line format repository definition.""" - return "{prefix}{repotype} {options}{uri} {release} {groups}".format( - prefix="" if self.enabled else "#", - repotype=self.repotype, - options=self.make_options_string(), - uri=self.uri, - release=self.release, - groups=" ".join(self.groups), - ) - @property def enabled(self): """Return whether or not the repository is enabled.""" @@ -1043,17 +987,6 @@ def filename(self, fname: str) -> None: raise InvalidSourceError("apt source filenames should end in .list or .sources!") self._filename = fname - def _make_filename(self) -> str: - """Construct a filename from uri and release. - - For internal use when a filename isn't set. - Should match the filename written to by add-apt-repository. - """ - return "{}-{}.list".format( - DebianRepository.prefix_from_uri(self.uri), - self.release.replace("/", "-"), - ) - @property def gpg_key(self): """Returns the path to the GPG key for this repository.""" @@ -1066,21 +999,17 @@ def options(self): """Returns any additional repo options which are set.""" return self._options - def make_options_string(self) -> str: + def make_options_string(self, include_signed_by: bool = True) -> str: """Generate the complete one-line-style options string for a repository. - Combining `gpg_key`, if set, and the rest of the options to find - a complex repo string. + Combining `gpg_key`, if set (and include_signed_by is True), with any other + provided options to form the options section of a one-line-style definition. """ options = self._options if self._options else {} - - gpg_key_filename = self.gpg_key # ensure getter logic is run - if gpg_key_filename: - options["signed-by"] = gpg_key_filename - + if include_signed_by and self.gpg_key: + options["signed-by"] = self.gpg_key if not options: return "" - pairs = ("{}={}".format(k, v) for k, v in sorted(options.items())) return "[{}] ".format(" ".join(pairs)) @@ -1100,18 +1029,28 @@ def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "Debian Args: repo_line: a string representing a repository entry write_file: boolean to enable writing the new repo to disk. True by default. - Results in calling `add-apt-repository --no-update --sourceslist $repo_line` + Expect it to result in an add-apt-repository call under the hood, like: + add-apt-repository --no-update --sourceslist="$repo_line" """ repo = RepositoryMapping._parse( # pyright: ignore[reportPrivateUsage] repo_line, filename="UserInput" # temp filename ) repo.filename = repo._make_filename() if write_file: - RepositoryMapping._add_apt_repository( # pyright: ignore[reportPrivateUsage]) - repo, update_cache=False - ) + _add_repository(repo) return repo + def _make_filename(self) -> str: + """Construct a filename from uri and release. + + For internal use when a filename isn't set. + Should match the filename written to by add-apt-repository. + """ + return "{}-{}.list".format( + DebianRepository.prefix_from_uri(self.uri), + self.release.replace("/", "-"), + ) + def disable(self) -> None: """Remove this repository by disabling it in the source file. @@ -1253,6 +1192,26 @@ def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: keyf.write(key_material) +def _repo_to_identifier(repo: DebianRepository) -> str: + """Return str identifier derived from repotype, uri, and release. + + Private method used to produce the identifiers used by RepositoryMapping. + """ + return "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) + + +def _repo_to_line(repo: DebianRepository, include_signed_by: bool = True) -> str: + """Return the one-per-line format repository definition.""" + return "{prefix}{repotype} {options}{uri} {release} {groups}".format( + prefix="" if repo.enabled else "#", + repotype=repo.repotype, + options=repo.make_options_string(include_signed_by=include_signed_by), + uri=repo.uri, + release=repo.release, + groups=" ".join(repo.groups), + ) + + class RepositoryMapping(Mapping[str, DebianRepository]): """An representation of known repositories. @@ -1341,19 +1300,15 @@ def load_deb822(self, filename: str) -> None: """ with open(filename, "r") as f: repos, errors = self._parse_deb822_lines(f, filename=filename) - for repo in repos: - identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] - self._repository_map[identifier] = repo - + self._repository_map[_repo_to_identifier(repo)] = repo if errors: - self._last_errors = errors + self._last_errors = tuple(errors) logger.debug( "the following %d error(s) were encountered when reading deb822 sources:\n%s", len(errors), "\n".join(str(e) for e in errors), ) - if repos: logger.info("parsed %d apt package repositories from %s", len(repos), filename) else: @@ -1468,95 +1423,29 @@ def _parse(line: str, filename: str) -> DebianRepository: raise InvalidSourceError("An invalid sources line was found in %s!", filename) def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented - self, - repo: DebianRepository, - default_filename: Optional[bool] = False, - update_cache: bool = False, - ) -> "RepositoryMapping": + self, repo: DebianRepository, default_filename: Optional[bool] = False + ) -> None: """Add a new repository to the system using add-apt-repository. Args: repo: a DebianRepository object where repo.enabled is True - update_cache: if True, apt-add-repository will update the package cache - and then a new RepositoryMapping object will be returned. - If False, then apt-add-repository is run with the --no-update option. - An entry for the repo is added to this RepositoryMapping before returning it. - Don't forget to call `apt.update` manually before installing any packages! - - Returns: - self, or a new RepositoryMapping object if update_cache is True - raises: ValueError: if repo.enabled is False CalledProcessError: if there's an error running apt-add-repository WARNING: Does not associate the repository with a signing key. - Use the `import_key` method to add a signing key globally. + Use `import_key` to add a signing key globally. + + WARNING: Don't forget to call `apt.update` before installing any packages! + Or call `apt.add_package` with `update_cache=True`. WARNING: the default_filename keyword argument is provided for backwards compatibility only. It is not used, and was not used in the previous revision of this library. """ - self._add_apt_repository(repo, update_cache=update_cache, remove=False) - if update_cache: - return RepositoryMapping() - identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] - self._repository_map[identifier] = repo - return self - - def _remove(self, repo: DebianRepository, update_cache: bool = False) -> "RepositoryMapping": - """Use add-apt-repository to remove a repository. - - Args: - repo: a DebianRepository object where repo.enabled is True - update_cache: if True, apt-add-repository will update the package cache - and then a new RepositoryMapping object will be returned. - If False, then apt-add-repository is run with the --no-update option. - Any entry for the repo is removed from this RepositoryMapping before returning it. - Don't forget to call `apt.update` manually before installing any packages! - - Returns: - self, or a new RepositoryMapping object if update_cache is True - - raises: - ValueError: if repo.enabled is False - CalledProcessError: if there's an error running apt-add-repository - """ - self._add_apt_repository(repo, update_cache=update_cache, remove=True) - if update_cache: - return RepositoryMapping() - identifier = repo._get_identifier() # pyright: ignore[reportPrivateUsage] - self._repository_map.pop(identifier, None) - return self - - @staticmethod - def _add_apt_repository( - repo: DebianRepository, - update_cache: bool = False, - remove: bool = False, - ) -> None: if not repo.enabled: raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) - line = repo._to_line() # pyright: ignore[reportPrivateUsage] - cmd = [ - "add-apt-repository", - "--yes", - "--sourceslist=" + line, - ] - if remove: - cmd.append("--remove") - if not update_cache: - cmd.append("--no-update") - logger.info("%s", cmd) - try: - subprocess.run(cmd, check=True, capture_output=True) - except CalledProcessError as e: - logger.error( - "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", - cmd, - e.stdout.decode(), - e.stderr.decode(), - ) - raise + _add_repository(repo) + self._repository_map[_repo_to_identifier(repo)] = repo def disable(self, repo: DebianRepository) -> None: """Remove a repository by disabling it in the source file. @@ -1567,6 +1456,44 @@ def disable(self, repo: DebianRepository) -> None: WARNING: This method does NOT alter the `.enabled` flag on the DebianRepository. """ repo.disable() + self._repository_map[_repo_to_identifier(repo)] = repo + # ^ adding to map on disable seems like a bug, but this is the previous behaviour + + +def _add_repository( + repo: DebianRepository, + remove: bool = False, + update_cache: bool = False, +) -> None: + line = _repo_to_line(repo, include_signed_by=False) + key_file = repo.gpg_key + if key_file and not os.path.exists(key_file): + msg = ( + "Adding repository '{line}' with add-apt-repository." + " Key file '{key_file}' does not exist." + " Ensure it is imported correctly to use this repository." + ).format(line=line, key_file=key_file) + logger.warning(msg) + cmd = [ + "add-apt-repository", + "--yes", + "--sourceslist=" + line, + ] + if remove: + cmd.append("--remove") + if not update_cache: + cmd.append("--no-update") + logger.info("%s", cmd) + try: + subprocess.run(cmd, check=True, capture_output=True) + except CalledProcessError as e: + logger.error( + "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", + cmd, + e.stdout.decode(), + e.stderr.decode(), + ) + raise class _Deb822Stanza: @@ -1613,6 +1540,35 @@ def get_gpg_key_filename(self) -> str: return self._gpg_key_filename +class MissingRequiredKeyError(InvalidSourceError): + """Missing a required value in a source file.""" + + def __init__(self, message: str = "", *, file: str, line: Optional[int], key: str) -> None: + super().__init__(message, file, line, key) + self.file = file + self.line = line + self.key = key + + +class BadValueError(InvalidSourceError): + """Bad value for an entry in a source file.""" + + def __init__( + self, + message: str = "", + *, + file: str, + line: Optional[int], + key: str, + value: str, + ) -> None: + super().__init__(message, file, line, key, value) + self.file = file + self.line = line + self.key = key + self.value = value + + def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]]]: """Given lines from a deb822 format file, yield a stanza of lines. @@ -1767,6 +1723,8 @@ def _deb822_options_to_repos( gpg_key_filename=gpg_key_file, options=options, ) - for repotype, uri, suite in itertools.product(repotypes, uris, suites) + for repotype in repotypes + for uri in uris + for suite in suites ) return repos, (gpg_key_file, gpg_key_from_stanza) diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index ea91ea62..42f9169f 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -51,102 +51,47 @@ def test_remove_package(): assert not get_command_path("cfssl") -def test_install_hardware_observer_ssacli(): - """Test the ability to install a package used by the hardware-observer charm. - - Here we follow the order of operations and arguments used in the charm: - for key in HP_KEYS: - apt.import_key(key) - - repositories = apt.RepositoryMapping() - repo = apt.DebianRepository.from_repo_line(...) - repositories.add(repo) - - apt.add_package(self.pkg, update_cache=True) - """ - line = "deb http://downloads.linux.hpe.com/SDR/repo/mcp stretch/current non-free" - repo_id = apt.DebianRepository.from_repo_line( - line, write_file=False - )._get_identifier() # pyright: ignore[reportPrivateUsage] - repos_before = apt.RepositoryMapping() - assert repo_id not in repos_before - assert not get_command_path("ssacli") - - key_files: List[str] = [] # just for cleanup - ## begin steps - for key in HP_KEYS: - key_file = apt.import_key(key) - key_files.append(key_file) - repositories = apt.RepositoryMapping() - repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default - # repo added to system but repositories doesn't know about it yet - assert repo_id not in repositories - repositories.add(repo) # update_cache=False by default - # `add` call is redundant with `from_repo_line` from system pov - # but it does add an entry to the RepositoryMapping - assert repo_id in repositories - apt.add_package("ssacli", update_cache=True) - assert get_command_path("ssacli") - # install succeed here as update_cache=True - ## end steps - - ## cleanup - for key_file in key_files: - os.remove(key_file) - repos_clean = repositories._remove( # pyright: ignore[reportPrivateUsage] - repo, update_cache=True - ) - assert repo_id not in repos_clean - apt.remove_package("ssacli") - assert not get_command_path("ssacli") - - def test_install_package_from_external_repository(): repo_id = "deb-https://repo.mongodb.org/apt/ubuntu-jammy/mongodb-org/8.0" - repos_before = apt.RepositoryMapping() - assert repo_id not in repos_before + repos = apt.RepositoryMapping() + assert repo_id not in repos assert not get_command_path("mongod") - + ## steps key = urlopen("https://www.mongodb.org/static/pgp/server-8.0.asc").read().decode() - line = "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/8.0 multiverse" - ## this is one valid order of operations: - ## apt.import_key(...) # before add - ## apt.RepositoryMapping.add(repo, update_cache=True) - ## apt.add_package(...) key_file = apt.import_key(key) + line = ( + "deb [ arch=amd64,arm64 signed-by={} ]" + " https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/8.0 multiverse" + ).format(key_file) ## if: use implicit write_file repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default - apt.update() - repos_after = apt.RepositoryMapping() + assert repo_id not in repos ## if: don't use implicit write_file # repo = apt.DebianRepository.from_repo_line(line, write_file=False) - # repos_after = repos_before.add(repo, update_cache=True) + # repos.add(repo) + # assert repo_id in repos ## fi - assert repo_id in repos_after + assert repo_id in apt.RepositoryMapping() + apt.update() apt.add_package("mongodb-org") assert get_command_path("mongod") subprocess.run(["mongod", "--version"], check=True) - ## cleanup os.remove(key_file) - repos_after._remove(repo, update_cache=False) # pyright: ignore[reportPrivateUsage] - assert repo_id not in repos_after - repos_before_update = apt.RepositoryMapping() - assert repo_id not in repos_before_update - # ^ although apt update hasn't been called yet, we've already modified files on disk + apt._add_repository(repo, remove=True) # pyright: ignore[reportPrivateUsage] + assert repo_id not in apt.RepositoryMapping() apt.update() - repos_clean = apt.RepositoryMapping() - assert repo_id not in repos_clean - apt.remove_package("mongodb-org", autoremove=True) # mongodb-org is a metapackage + apt.remove_package("mongodb-org") + assert get_command_path("mongod") # mongodb-org is a metapackage + subprocess.run(["apt", "autoremove"], check=True) assert not get_command_path("mongod") def test_install_higher_version_package_from_external_repository(): repo_id = "deb-https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/-jammy" - repos_before = apt.RepositoryMapping() - assert repo_id not in repos_before - - # version before + repos = apt.RepositoryMapping() + assert repo_id not in repos + ## version before if not get_command_path("inkscape"): apt.add_package("inkscape") version_before = subprocess.run( @@ -157,7 +102,7 @@ def test_install_higher_version_package_from_external_repository(): ).stdout apt.remove_package("inkscape") assert not get_command_path("inkscape") - + ## steps repo = apt.DebianRepository( enabled=True, repotype="deb", @@ -165,13 +110,9 @@ def test_install_higher_version_package_from_external_repository(): release="jammy", groups=["main"], ) - ## this is a different, valid order of operations: - ## apt.RepositoryMapping.add(..., update_cache=False) - ## apt.import_key(...) # before update but after add - ## apt.update() - ## apt.add_package(...) - repos_after = repos_before.add(repo) # update_cache=False by default - assert repo_id in repos_after + repos.add(repo) # update_cache=False by default + assert repo_id in repos + assert repo_id in apt.RepositoryMapping() key_file = apt.import_key(INKSCAPE_KEY) apt.update() apt.add_package("inkscape") @@ -183,17 +124,61 @@ def test_install_higher_version_package_from_external_repository(): text=True, ).stdout assert version_after > version_before # lexical comparison :( - ## cleanup os.remove(key_file) - repos_clean = repos_after._remove( # pyright: ignore[reportPrivateUsage] - repo, update_cache=True - ) - assert repo_id not in repos_clean + apt._add_repository(repo, remove=True) # pyright: ignore[reportPrivateUsage] + assert repo_id not in apt.RepositoryMapping() + apt.update() apt.remove_package("inkscape") assert not get_command_path("inkscape") +def test_install_hardware_observer_ssacli(): + """Test the ability to install a package used by the hardware-observer charm. + + Here we follow the order of operations and arguments used in the charm: + for key in HP_KEYS: + apt.import_key(key) + + repositories = apt.RepositoryMapping() + repo = apt.DebianRepository.from_repo_line(...) + repositories.add(repo) + + apt.add_package(self.pkg, update_cache=True) + """ + line = "deb http://downloads.linux.hpe.com/SDR/repo/mcp stretch/current non-free" + repo_id = apt._repo_to_identifier( # pyright: ignore[reportPrivateUsage] + apt.DebianRepository.from_repo_line(line, write_file=False) + ) + assert repo_id not in apt.RepositoryMapping() + assert not get_command_path("ssacli") + key_files: List[str] = [] # just for cleanup + ## steps + for key in HP_KEYS: + key_file = apt.import_key(key) + key_files.append(key_file) + repos = apt.RepositoryMapping() + repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default + assert repo_id in apt.RepositoryMapping() + # repo added to system but repos doesn't know about it yet + assert repo_id not in repos + repos.add(repo) + assert repo_id in repos + # `add` call is redundant with `from_repo_line` from system pov + # but adds an entry to the RepositoryMapping + apt.add_package("ssacli", update_cache=True) + # apt.update not required since update_cache=True + assert get_command_path("ssacli") + ## cleanup + for key_file in key_files: + os.remove(key_file) + apt._add_repository(repo, remove=True) # pyright: ignore[reportPrivateUsage] + assert repo_id not in apt.RepositoryMapping() + apt.update() + apt.remove_package("ssacli") + assert not get_command_path("ssacli") + + def test_from_apt_cache_error(): try: apt.DebianPackage.from_apt_cache("ceci-n'est-pas-un-paquet") @@ -232,7 +217,6 @@ def test_from_apt_cache_error(): =VF33 -----END PGP PUBLIC KEY BLOCK----- """ -"""Static parameters for keys.""" HPPUBLICKEY1024 = """ -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index 0bad2df5..df2142e7 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -209,7 +209,7 @@ def test_load_deb822_exact_path(repo_mapping: apt.RepositoryMapping): [repo] = repo_mapping assert isinstance(repo, apt.DebianRepository) assert repo.uri - assert "/" in repo.release + assert repo.release.endswith("/") assert not repo.groups @@ -317,33 +317,11 @@ def test_add_with_deb822(repo_mapping: apt.RepositoryMapping): assert len(repos) == 1 assert not errors [repo] = repos - identifier = repo._get_identifier() - # call add with update_cache=True + identifier = apt._repo_to_identifier(repo) with patch.object(apt.subprocess, "run") as mock_run_1: - with patch.object( - apt.RepositoryMapping, - "_apt_dir", - str(FAKE_APT_DIRS / "empty"), - ): - r1 = repo_mapping.add(repo, update_cache=True) - assert r1 is not repo_mapping - assert identifier not in repo_mapping # because it's the old mapping - assert identifier not in r1 # because we mocked out the subprocess call - mock_run_1.assert_called_once_with( - [ - "add-apt-repository", - "--yes", - "--sourceslist=deb http://nz.archive.ubuntu.com/ubuntu/ an/exact/path/ ", - ], - check=True, - capture_output=True, - ) - # call add with update_cache=False - with patch.object(apt.subprocess, "run") as mock_run_2: - r2 = repo_mapping.add(repo) # update_cache=False by default - assert r2 is repo_mapping + repo_mapping.add(repo) assert identifier in repo_mapping - mock_run_2.assert_called_once_with( + mock_run_1.assert_called_once_with( [ "add-apt-repository", "--yes", diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index dc4c6e1b..164a9c83 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -106,7 +106,7 @@ def test_can_add_repositories(self): open(d.filename).readlines(), ) - def test_can_add_repositories_from_string(self): + def test_can_create_repo_from_repo_line(self): d = apt.DebianRepository.from_repo_line( "deb https://example.com/foo focal bar baz", write_file=False, @@ -117,8 +117,6 @@ def test_can_add_repositories_from_string(self): self.assertEqual(d.release, "focal") self.assertEqual(d.groups, ["bar", "baz"]) self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") - ## FIXME: need integration test for write_file=True - # self.assertIn("deb https://example.com/foo focal bar baz\n", open(d.filename).readlines()) @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") def test_valid_list_file(self): @@ -150,8 +148,3 @@ def test_can_add_repositories_from_string_with_options(self): self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") self.assertEqual(d.gpg_key, "/foo/gpg.key") self.assertEqual(d.options["arch"], "amd64") - ## FIXME: need integration test for write_file=True - # self.assertIn( - # "deb [arch=amd64 signed-by=/foo/gpg.key] https://example.com/foo focal bar baz\n", - # open(d.filename).readlines(), - # ) From d0674e9619e63129bd5fe3f75ab6879f4f932fdc Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 17:18:41 +1300 Subject: [PATCH 43/50] refactor: move keys to files --- .../keys/HPEPUBLICKEY2048_KEY1.asc | 19 +++ tests/integration/keys/HPPUBLICKEY1024.asc | 30 ++++ tests/integration/keys/HPPUBLICKEY2048.asc | 19 +++ .../integration/keys/HPPUBLICKEY2048_KEY1.asc | 19 +++ tests/integration/keys/INKSCAPE_KEY.asc | 29 ++++ tests/integration/test_apt.py | 153 ++---------------- 6 files changed, 127 insertions(+), 142 deletions(-) create mode 100644 tests/integration/keys/HPEPUBLICKEY2048_KEY1.asc create mode 100644 tests/integration/keys/HPPUBLICKEY1024.asc create mode 100644 tests/integration/keys/HPPUBLICKEY2048.asc create mode 100644 tests/integration/keys/HPPUBLICKEY2048_KEY1.asc create mode 100644 tests/integration/keys/INKSCAPE_KEY.asc diff --git a/tests/integration/keys/HPEPUBLICKEY2048_KEY1.asc b/tests/integration/keys/HPEPUBLICKEY2048_KEY1.asc new file mode 100644 index 00000000..5551e650 --- /dev/null +++ b/tests/integration/keys/HPEPUBLICKEY2048_KEY1.asc @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +mQENBFZp0LkBCACXajRw3b4x7G7dulNYj0hUID4BtVFq/MjEb6PHckTxGxZDoQRX +RK54tiTFA9wq3b4P3yEFnOjbjRoI0d7Ls67FADugFO+cDCtsV9yuDlaYP/U/h2nX +N0R4AdYbsVd5yr6xr+GAy66Hmx5jFH3kbC+zJpOcI0tU9hcyU7gjbxu6KQ1ypI2Q +VRKf8sRBJXgmkOlbYx35ZUMFcmVxrLJXvUuxmAVXgT9f5M3Z3rsGt/ab+/+1TFSb +RsaqHsIPE0QH8ikqW4IeDQAo1T99pCdf7FWr45KFFTo7O4AZdLMWVgqeFHaSoZxJ +307VIINsWiwQoPp0tfU5NOOOwB1Sv3x9QgFtABEBAAG0P0hld2xldHQgUGFja2Fy +ZCBFbnRlcnByaXNlIENvbXBhbnkgUlNBLTIwNDgtMjUgPHNpZ25ocEBocGUuY29t +PokBPQQTAQIAJwUCVmnQuQIbLwUJEswDAAYLCQgHAwIGFQgCCQoLAxYCAQIeAQIX +gAAKCRDCCK3eJsK3l9G+B/0ekblsBeN+xHIJ28pvo2aGb2KtWBwbT1ugI+aIS17K +UQyHZJUQH+ZeRLvosuoiQEdcGIqmOxi2hVhSCQAOV1LAonY16ACveA5DFAEBz1+a +WQyx6sOLLEAVX1VqGlBXxh3XLEUWOhlAf1gZPNtHsmURTUy2h1Lv/Yoj8KLyuK2n +DmrLOS3Ro+RqWocaJfvAgXKgt6Fq/ChDUHOnar7lGswzMsbE/yzLJ7He4y89ImK+ +2ktR5HhDuxqgCe9CWH6Q/1WGhUa0hZ3nbluq7maa+kPe2g7JcRzPH/nJuDCAOZ7U +6mHE8j0kMQMYjgaYEx2wc02aQRmPyxhbDLjSbtjomXRr +=voON +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/keys/HPPUBLICKEY1024.asc b/tests/integration/keys/HPPUBLICKEY1024.asc new file mode 100644 index 00000000..c872555d --- /dev/null +++ b/tests/integration/keys/HPPUBLICKEY1024.asc @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.0 (MingW32) + +mQGiBEIxWpoRBADb06sJgnD7MJnm2Ny1nmTFLDSZ8vkubP+pmfn9N9TE26oit+KI +OnVTRVbSPl3F15wTjSBGR453MEfnzp1NrMk1GIa/m1nKAmgQ4t1714C4jQab0to+ +gP51XhPhtAGt7BggorQw2RXa4KdTCh8ByOIaDKRYcESmMazSZ+Pscy2XRwCgm771 +21RCM0RcG2dmHZZgKH8fTscD/RiY3CHI2jJl9WosIYXbZpOySzrLn0lRCRdNdpew +Y5m1f3lhqoSvJk7pXjs4U+3XlOlUhgWl5HiXuWSVyPu2ilfGdfgpJslawI85fBQg +Ul5kcrjLHHsApeG8oGStFJE2JAc+0D+whmGmJbjWKwuZJmgpm9INplA4h1BYJbx+ +6A3MBACFiMTttDPpJ+5eWr1VSZwxCZNqvPWmjpL5Nh9F8xzE7q+ad2CFKSebvRrv +Jf7Y2m+wY9bmo5nJ3wHYEX3Aatt+QVF10G6wTdIz/Ohm/Pc4Li4NhzYOv7FKxVam +97UN0O8Rsl4GhE2eE8H+Q3QYFvknAWoTj3Rq3/A5FA6FsRFhxbQwSGV3bGV0dC1Q +YWNrYXJkIENvbXBhbnkgKEhQIENvZGVzaWduaW5nIFNlcnZpY2UpiGQEExECACQF +AkIxWpoCGwMFCRLMAwAGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQUnvFOiaJuIc1 +2wCgj2UotUgSegPHmcKdApY+4WFaz/QAnjI58l5bDD8eElBCErHVoq9uPMczuQIN +BEIxWqUQCADnBXqoU8QeZPEy38oI0GrN2q7nvS+4UBQeIRVy8x+cOqDRDcE8PHej +7NtxP698U0WFGK47GszjiV4WTnvexuJk0B5AMEBHana8fVj7uRUcmyYZqOZd7EXn +Q3Ivi8itfkTICkhZi7bmGsSF0iJ0eAI5n2bCqJykNQvJ6a3dWJKP8EgaBCZj+TGL +WWJHDZsrn8g4BeaNS/MbmsCLAk8N6bWMGzAKfgxUraMCwuZ9fVyHFavHdeChUtna +qnF4uw0hHLaGWmTJjziXVvVC1a8+inTxPZkVpAvD0A+/LNlkP7TtAdaVOJqv3+a3 +ybMQL851bRTFyt+H0XGHhzhhtuu9+DyfAAMFCADRWGxIfniVG7O4wtwLD3sWzR/W +LmFlJYu4s9rSDgn3NDjigQzZoVtbuv3Z9IZxBMoYa50MuybuVDp55z/wmxvYoW2G +25kOFDKx/UmkKkUBLdokb5V1p9j5SJorGBSfsNAHflhmBhyuMP4CDISbBUSN7oO1 +Oj41jNxpqhy+8ayygSVcTNwMe909J/HdC//xFANLDhjKPf3ZAulWNhOvjTlpF46B +yt1l8ZNinIeE7CFL7H+LlMl2Ml6wsOkrxsSauBis6nER4sYVqrMdzpUU2Sr2hj6Q +sJ+9TS+IURcnxL/M851KCwLhwZKdphQjT3mXXsoCx/l3rI6cxpwYgjiKiZhOiE8E +GBECAA8FAkIxWqUCGwwFCRLMAwAACgkQUnvFOiaJuIenewCdHcEvMxBYprqRjKUw +04EypyFtZTgAn0wds0nbpd2+VZ5WHbVRfU4y5Y5Y +=+cX+ +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/keys/HPPUBLICKEY2048.asc b/tests/integration/keys/HPPUBLICKEY2048.asc new file mode 100644 index 00000000..36202be3 --- /dev/null +++ b/tests/integration/keys/HPPUBLICKEY2048.asc @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (MingW32) + +mQENBFC+QboBCAC1bodHD7AmR00SkDMB4u9MXy+Z5vv8wbmGRaKDBYScpAknOljX +d5tBADffAetd1hgLnrLKN8vHdIsYkmUyeEeEsnIUKtwvbx/f6PoZZPOIIIRh1d2W +Mjw9qXIE+tgr2gWlq0Gi5BZzaKse1+khRQ2rewJBppblSGWgcmCMIq8OwAsrdbtr +z7+37c/g/Y2VfAahc23YZW9LQ5MiaI4nS4JMZbWPYtBdF78B/D2t5FvmvDG0Cgjk +Qi1U9IVjiFKixuoi6nRsvBLFYL/cI+vo4iyUC5x7qmKd8gN7A030gS67VrleNRki +q0vaF6J46XpIl4o58t23FSAKKRbTwavYzdMpABEBAAG0NEhld2xldHQtUGFja2Fy +ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSmJAT4EEwECACgF +AlC+QboCGwMFCRLMAwAGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELBwaApc +4tR2x7sH/A3D4XxEEyrX6Z3HeWSSA80+n+r5QwfXm5unxsWEL3JyNg6sojlrJY4K +8k4ih4nkY4iblChTCSQwnqKXqkL5U+RIr+AJoPx+55M98u4eRTVYMHZD7/jFq85z +ZFGUkFkars9E2aRzWhqbz0LINb9OUeX0tT5qQseHflO2PaJykxNPC14WhsBKC2lg +dZWnGhO5QJFp69AnSp4k+Uo/1LMk87YEJIL1NDR0lrlKgRvFfFyTpRBt+Qb1Bb7g +rjN0171g8t5GaPWamN3Oua/v4aZg15f3xydRF8y9TsYjiNz+2TzRjKv7AkpZaJST +06CqMjCgiZ6UFFGN0/oqLnwxdP3Mmh4= +=aphN +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/keys/HPPUBLICKEY2048_KEY1.asc b/tests/integration/keys/HPPUBLICKEY2048_KEY1.asc new file mode 100644 index 00000000..5eb30d4f --- /dev/null +++ b/tests/integration/keys/HPPUBLICKEY2048_KEY1.asc @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (MingW32) + +mQENBFRtGAgBCADlSku65P14hVdx9E/W0n6MwuB3WGqmsyKNoa3HezFdMjWERldI +NNUdi8O28cZ6j2+Hi9L1HeQIQ9+7FHpR3JyQePBJtRX8WSEusfRtML98opDhJxKm +8Jyxb7aTvCwdNHz3yxADINkMtOj5oRm7VCr8XHkG7YU27ELs8B+BXWvjO21oSosi +FurnhT+H3hQsYXfYA55aa21q0qX+L5dFJSNdzZVo7m9ybioVv2R5+PfBvdaSxCnm +OpcGXFaKAsqVHeTW0pd3sdkin1rkbhOBaU5lFBt2ZiMtKpKHpT8TZnqHpFHFbgi8 +j2ARJj4IDct2OGILddUIZSFyue6WE2hpV5c/ABEBAAG0OEhld2xldHQtUGFja2Fy +ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSkgLSAxiQE+BBMB +AgAoBQJUbRgIAhsDBQkSzAMABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRD6 +3Y1ksSdeo6BJCADOfIPPLPpIOnFK9jH4t8lLUd+RyMc+alA3uTDPUJa/ZHa6DHfh +42iaPYVEV8OG0tnbMlHmwvsZ5c1/MRMw1UbxCvD88P2qM4SUrUjQUlSCms2GLGvF +ftFXBiOJQ7/yBc9o+yoSvwPrrTxSCk4+Sqm0IfVXVzChDM9dM9YPY2Vzjd+LUaYC +3X+eSuggUDO0TmJLJd7tZdF9fVXq3lr63BZ5PY98MTCuOoeSMDa9FIUQf6vn6UUJ +MDSRZ9OzhpNJOKR+ShVRwDK6My8gtVIW1EAW2w3VQWI2UNF07aLeO8UG6nTNWA23 ++OuZkUdgQovjcq01caSefgOkmiQOx6d74CAk +=X+eo +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/keys/INKSCAPE_KEY.asc b/tests/integration/keys/INKSCAPE_KEY.asc new file mode 100644 index 00000000..0c5637b5 --- /dev/null +++ b/tests/integration/keys/INKSCAPE_KEY.asc @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU +1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM +3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 +vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U +NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu +5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP +pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww +2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e +ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA +RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ +S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB +tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 +FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC +AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle +amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO +qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu +oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R +PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE +MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom +4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 +J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ +q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z +lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k +uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO +2ekz6IVTXVA= +=VF33 +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index 42f9169f..ec753d62 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -6,6 +6,7 @@ import logging import os import subprocess +from pathlib import Path from typing import List from urllib.request import urlopen @@ -14,6 +15,8 @@ logger = logging.getLogger(__name__) +KEY_DIR = Path(__file__).parent / "keys" + def test_install_packages(): apt.update() @@ -113,7 +116,7 @@ def test_install_higher_version_package_from_external_repository(): repos.add(repo) # update_cache=False by default assert repo_id in repos assert repo_id in apt.RepositoryMapping() - key_file = apt.import_key(INKSCAPE_KEY) + key_file = apt.import_key((KEY_DIR / "INKSCAPE_KEY.asc").read_text()) apt.update() apt.add_package("inkscape") assert get_command_path("inkscape") @@ -154,8 +157,13 @@ def test_install_hardware_observer_ssacli(): assert not get_command_path("ssacli") key_files: List[str] = [] # just for cleanup ## steps - for key in HP_KEYS: - key_file = apt.import_key(key) + for path in ( + KEY_DIR / "HPEPUBLICKEY2048_KEY1.asc", + KEY_DIR / "HPPUBLICKEY2048_KEY1.asc", + KEY_DIR / "HPPUBLICKEY2048.asc", + KEY_DIR / "HPPUBLICKEY1024.asc", + ): + key_file = apt.import_key(path.read_text()) key_files.append(key_file) repos = apt.RepositoryMapping() repo = apt.DebianRepository.from_repo_line(line) # write_file=True by default @@ -184,142 +192,3 @@ def test_from_apt_cache_error(): apt.DebianPackage.from_apt_cache("ceci-n'est-pas-un-paquet") except apt.PackageError as e: assert "No packages found" in str(e) - - -INKSCAPE_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- - . - mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU - 1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM - 3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 - vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U - NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu - 5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP - pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww - 2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e - ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA - RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ - S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB - tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 - FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC - AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle - amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO - qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu - oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R - PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE - MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom - 4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 - J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ - q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z - lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k - uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO - 2ekz6IVTXVA= - =VF33 - -----END PGP PUBLIC KEY BLOCK----- -""" - -HPPUBLICKEY1024 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.0 (MingW32) - -mQGiBEIxWpoRBADb06sJgnD7MJnm2Ny1nmTFLDSZ8vkubP+pmfn9N9TE26oit+KI -OnVTRVbSPl3F15wTjSBGR453MEfnzp1NrMk1GIa/m1nKAmgQ4t1714C4jQab0to+ -gP51XhPhtAGt7BggorQw2RXa4KdTCh8ByOIaDKRYcESmMazSZ+Pscy2XRwCgm771 -21RCM0RcG2dmHZZgKH8fTscD/RiY3CHI2jJl9WosIYXbZpOySzrLn0lRCRdNdpew -Y5m1f3lhqoSvJk7pXjs4U+3XlOlUhgWl5HiXuWSVyPu2ilfGdfgpJslawI85fBQg -Ul5kcrjLHHsApeG8oGStFJE2JAc+0D+whmGmJbjWKwuZJmgpm9INplA4h1BYJbx+ -6A3MBACFiMTttDPpJ+5eWr1VSZwxCZNqvPWmjpL5Nh9F8xzE7q+ad2CFKSebvRrv -Jf7Y2m+wY9bmo5nJ3wHYEX3Aatt+QVF10G6wTdIz/Ohm/Pc4Li4NhzYOv7FKxVam -97UN0O8Rsl4GhE2eE8H+Q3QYFvknAWoTj3Rq3/A5FA6FsRFhxbQwSGV3bGV0dC1Q -YWNrYXJkIENvbXBhbnkgKEhQIENvZGVzaWduaW5nIFNlcnZpY2UpiGQEExECACQF -AkIxWpoCGwMFCRLMAwAGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQUnvFOiaJuIc1 -2wCgj2UotUgSegPHmcKdApY+4WFaz/QAnjI58l5bDD8eElBCErHVoq9uPMczuQIN -BEIxWqUQCADnBXqoU8QeZPEy38oI0GrN2q7nvS+4UBQeIRVy8x+cOqDRDcE8PHej -7NtxP698U0WFGK47GszjiV4WTnvexuJk0B5AMEBHana8fVj7uRUcmyYZqOZd7EXn -Q3Ivi8itfkTICkhZi7bmGsSF0iJ0eAI5n2bCqJykNQvJ6a3dWJKP8EgaBCZj+TGL -WWJHDZsrn8g4BeaNS/MbmsCLAk8N6bWMGzAKfgxUraMCwuZ9fVyHFavHdeChUtna -qnF4uw0hHLaGWmTJjziXVvVC1a8+inTxPZkVpAvD0A+/LNlkP7TtAdaVOJqv3+a3 -ybMQL851bRTFyt+H0XGHhzhhtuu9+DyfAAMFCADRWGxIfniVG7O4wtwLD3sWzR/W -LmFlJYu4s9rSDgn3NDjigQzZoVtbuv3Z9IZxBMoYa50MuybuVDp55z/wmxvYoW2G -25kOFDKx/UmkKkUBLdokb5V1p9j5SJorGBSfsNAHflhmBhyuMP4CDISbBUSN7oO1 -Oj41jNxpqhy+8ayygSVcTNwMe909J/HdC//xFANLDhjKPf3ZAulWNhOvjTlpF46B -yt1l8ZNinIeE7CFL7H+LlMl2Ml6wsOkrxsSauBis6nER4sYVqrMdzpUU2Sr2hj6Q -sJ+9TS+IURcnxL/M851KCwLhwZKdphQjT3mXXsoCx/l3rI6cxpwYgjiKiZhOiE8E -GBECAA8FAkIxWqUCGwwFCRLMAwAACgkQUnvFOiaJuIenewCdHcEvMxBYprqRjKUw -04EypyFtZTgAn0wds0nbpd2+VZ5WHbVRfU4y5Y5Y -=+cX+ ------END PGP PUBLIC KEY BLOCK----- -""" - -HPPUBLICKEY2048 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (MingW32) - -mQENBFC+QboBCAC1bodHD7AmR00SkDMB4u9MXy+Z5vv8wbmGRaKDBYScpAknOljX -d5tBADffAetd1hgLnrLKN8vHdIsYkmUyeEeEsnIUKtwvbx/f6PoZZPOIIIRh1d2W -Mjw9qXIE+tgr2gWlq0Gi5BZzaKse1+khRQ2rewJBppblSGWgcmCMIq8OwAsrdbtr -z7+37c/g/Y2VfAahc23YZW9LQ5MiaI4nS4JMZbWPYtBdF78B/D2t5FvmvDG0Cgjk -Qi1U9IVjiFKixuoi6nRsvBLFYL/cI+vo4iyUC5x7qmKd8gN7A030gS67VrleNRki -q0vaF6J46XpIl4o58t23FSAKKRbTwavYzdMpABEBAAG0NEhld2xldHQtUGFja2Fy -ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSmJAT4EEwECACgF -AlC+QboCGwMFCRLMAwAGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELBwaApc -4tR2x7sH/A3D4XxEEyrX6Z3HeWSSA80+n+r5QwfXm5unxsWEL3JyNg6sojlrJY4K -8k4ih4nkY4iblChTCSQwnqKXqkL5U+RIr+AJoPx+55M98u4eRTVYMHZD7/jFq85z -ZFGUkFkars9E2aRzWhqbz0LINb9OUeX0tT5qQseHflO2PaJykxNPC14WhsBKC2lg -dZWnGhO5QJFp69AnSp4k+Uo/1LMk87YEJIL1NDR0lrlKgRvFfFyTpRBt+Qb1Bb7g -rjN0171g8t5GaPWamN3Oua/v4aZg15f3xydRF8y9TsYjiNz+2TzRjKv7AkpZaJST -06CqMjCgiZ6UFFGN0/oqLnwxdP3Mmh4= -=aphN ------END PGP PUBLIC KEY BLOCK----- -""" - -HPPUBLICKEY2048_KEY1 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.12 (MingW32) - -mQENBFRtGAgBCADlSku65P14hVdx9E/W0n6MwuB3WGqmsyKNoa3HezFdMjWERldI -NNUdi8O28cZ6j2+Hi9L1HeQIQ9+7FHpR3JyQePBJtRX8WSEusfRtML98opDhJxKm -8Jyxb7aTvCwdNHz3yxADINkMtOj5oRm7VCr8XHkG7YU27ELs8B+BXWvjO21oSosi -FurnhT+H3hQsYXfYA55aa21q0qX+L5dFJSNdzZVo7m9ybioVv2R5+PfBvdaSxCnm -OpcGXFaKAsqVHeTW0pd3sdkin1rkbhOBaU5lFBt2ZiMtKpKHpT8TZnqHpFHFbgi8 -j2ARJj4IDct2OGILddUIZSFyue6WE2hpV5c/ABEBAAG0OEhld2xldHQtUGFja2Fy -ZCBDb21wYW55IFJTQSAoSFAgQ29kZXNpZ25pbmcgU2VydmljZSkgLSAxiQE+BBMB -AgAoBQJUbRgIAhsDBQkSzAMABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRD6 -3Y1ksSdeo6BJCADOfIPPLPpIOnFK9jH4t8lLUd+RyMc+alA3uTDPUJa/ZHa6DHfh -42iaPYVEV8OG0tnbMlHmwvsZ5c1/MRMw1UbxCvD88P2qM4SUrUjQUlSCms2GLGvF -ftFXBiOJQ7/yBc9o+yoSvwPrrTxSCk4+Sqm0IfVXVzChDM9dM9YPY2Vzjd+LUaYC -3X+eSuggUDO0TmJLJd7tZdF9fVXq3lr63BZ5PY98MTCuOoeSMDa9FIUQf6vn6UUJ -MDSRZ9OzhpNJOKR+ShVRwDK6My8gtVIW1EAW2w3VQWI2UNF07aLeO8UG6nTNWA23 -+OuZkUdgQovjcq01caSefgOkmiQOx6d74CAk -=X+eo ------END PGP PUBLIC KEY BLOCK----- -""" - -HPEPUBLICKEY2048_KEY1 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.12 (GNU/Linux) - -mQENBFZp0LkBCACXajRw3b4x7G7dulNYj0hUID4BtVFq/MjEb6PHckTxGxZDoQRX -RK54tiTFA9wq3b4P3yEFnOjbjRoI0d7Ls67FADugFO+cDCtsV9yuDlaYP/U/h2nX -N0R4AdYbsVd5yr6xr+GAy66Hmx5jFH3kbC+zJpOcI0tU9hcyU7gjbxu6KQ1ypI2Q -VRKf8sRBJXgmkOlbYx35ZUMFcmVxrLJXvUuxmAVXgT9f5M3Z3rsGt/ab+/+1TFSb -RsaqHsIPE0QH8ikqW4IeDQAo1T99pCdf7FWr45KFFTo7O4AZdLMWVgqeFHaSoZxJ -307VIINsWiwQoPp0tfU5NOOOwB1Sv3x9QgFtABEBAAG0P0hld2xldHQgUGFja2Fy -ZCBFbnRlcnByaXNlIENvbXBhbnkgUlNBLTIwNDgtMjUgPHNpZ25ocEBocGUuY29t -PokBPQQTAQIAJwUCVmnQuQIbLwUJEswDAAYLCQgHAwIGFQgCCQoLAxYCAQIeAQIX -gAAKCRDCCK3eJsK3l9G+B/0ekblsBeN+xHIJ28pvo2aGb2KtWBwbT1ugI+aIS17K -UQyHZJUQH+ZeRLvosuoiQEdcGIqmOxi2hVhSCQAOV1LAonY16ACveA5DFAEBz1+a -WQyx6sOLLEAVX1VqGlBXxh3XLEUWOhlAf1gZPNtHsmURTUy2h1Lv/Yoj8KLyuK2n -DmrLOS3Ro+RqWocaJfvAgXKgt6Fq/ChDUHOnar7lGswzMsbE/yzLJ7He4y89ImK+ -2ktR5HhDuxqgCe9CWH6Q/1WGhUa0hZ3nbluq7maa+kPe2g7JcRzPH/nJuDCAOZ7U -6mHE8j0kMQMYjgaYEx2wc02aQRmPyxhbDLjSbtjomXRr -=voON ------END PGP PUBLIC KEY BLOCK----- -""" - -HP_KEYS = ( - HPEPUBLICKEY2048_KEY1, - HPPUBLICKEY2048_KEY1, - HPPUBLICKEY2048, - HPPUBLICKEY1024, -) From 3745426e7712674cd54c6de943283536fc990d42 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 17:42:25 +1300 Subject: [PATCH 44/50] fix: don't warn about key file on remove --- lib/charms/operator_libs_linux/v0/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 6d14514f..92fa63de 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1467,7 +1467,7 @@ def _add_repository( ) -> None: line = _repo_to_line(repo, include_signed_by=False) key_file = repo.gpg_key - if key_file and not os.path.exists(key_file): + if key_file and not remove and not os.path.exists(key_file): msg = ( "Adding repository '{line}' with add-apt-repository." " Key file '{key_file}' does not exist." From 865aad85a519284734584b5641400410b4207ac0 Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 17:47:27 +1300 Subject: [PATCH 45/50] style: use tuple for endswith --- lib/charms/operator_libs_linux/v0/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index 92fa63de..a6cf907c 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -983,7 +983,7 @@ def filename(self, fname: str) -> None: Args: fname: a filename to write the repository information to. """ - if not fname.endswith(".list") or fname.endswith(".sources"): + if not fname.endswith((".list", ".sources")): raise InvalidSourceError("apt source filenames should end in .list or .sources!") self._filename = fname From b993c54fdd0065e92156c50d4f68ca892e416c1b Mon Sep 17 00:00:00 2001 From: James Garner Date: Wed, 4 Dec 2024 17:51:57 +1300 Subject: [PATCH 46/50] style: paragraph -> stanza, repositories -> repos --- lib/charms/operator_libs_linux/v0/apt.py | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index a6cf907c..bf2fe98a 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1289,7 +1289,7 @@ def load_deb822(self, filename: str) -> None: """Load a deb822 format repository source file into the cache. In contrast to one-line-style, the deb822 format specifies a repository - using a multi-line paragraph. Paragraphs are separated by whitespace, + using a multi-line stanza. Stanzas are separated by whitespace, and each definition consists of lines that are either key: value pairs, or continuations of the previous value. @@ -1327,7 +1327,7 @@ def _parse_deb822_lines( `_parse_deb822_lines` strips out comments entirely when parsing a file into stanzas, instead only reading the 'Enabled' key to determine if an entry is enabled """ - repositories: List[DebianRepository] = [] + repos: List[DebianRepository] = [] errors: List[InvalidSourceError] = [] for numbered_lines in _iter_deb822_stanzas(lines): try: @@ -1335,8 +1335,8 @@ def _parse_deb822_lines( except InvalidSourceError as e: errors.append(e) else: - repositories.extend(stanza.repositories) - return repositories, errors + repos.extend(stanza.repos) + return repos, errors def load(self, filename: str): """Load a one-line-style format repository source file into the cache. @@ -1506,7 +1506,7 @@ def __init__(self, numbered_lines: List[Tuple[int, str]], filename: str = ""): self._filename = filename self._numbered_lines = numbered_lines if not numbered_lines: - self._repositories = () + self._repos = () self._gpg_key_filename = "" self._gpg_key_from_stanza = None return @@ -1516,13 +1516,13 @@ def __init__(self, numbered_lines: List[Tuple[int, str]], filename: str = ""): ) for repo in repos: repo._deb822_stanza = self # pyright: ignore[reportPrivateUsage] - self._repositories = repos + self._repos = repos self._gpg_key_filename, self._gpg_key_from_stanza = gpg_key_info @property - def repositories(self) -> Tuple[DebianRepository, ...]: + def repos(self) -> Tuple[DebianRepository, ...]: """The repositories defined by this deb822 stanza.""" - return self._repositories + return self._repos def get_gpg_key_filename(self) -> str: """Return the path to the GPG key for this stanza. @@ -1579,18 +1579,18 @@ def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[List[Tuple[int, str]] lists of numbered lines (a tuple of line number and line) that make up a deb822 stanza, with comments stripped out (but accounted for in line numbering) """ - current_paragraph: List[Tuple[int, str]] = [] + current_stanza: List[Tuple[int, str]] = [] for n, line in enumerate(lines, start=1): # 1 indexed line numbers - if not line.strip(): # blank lines separate paragraphs - if current_paragraph: - yield current_paragraph - current_paragraph = [] + if not line.strip(): # blank lines separate stanzas + if current_stanza: + yield current_stanza + current_stanza = [] continue content, _delim, _comment = line.partition("#") if content.strip(): # skip (potentially indented) comment line - current_paragraph.append((n, content.rstrip())) # preserve indent - if current_paragraph: - yield current_paragraph + current_stanza.append((n, content.rstrip())) # preserve indent + if current_stanza: + yield current_stanza def _deb822_stanza_to_options( @@ -1700,7 +1700,7 @@ def _deb822_options_to_repos( msg = ( "Since 'Suites' (line {suites_line}) does not specify" " a path relative to 'URIs' (line {uris_line})," - " 'Components' must be present in this paragraph." + " 'Components' must be present in this stanza." ).format( suites_line=line_numbers.get("Suites"), uris_line=line_numbers.get("URIs"), From 9ce3e4872e412b6498675735c5a1b0494bbf8160 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 5 Dec 2024 12:46:01 +1300 Subject: [PATCH 47/50] test: switch from inkscape to terminator ppa for ci --- tests/integration/keys/INKSCAPE_KEY.asc | 29 ----------------------- tests/integration/keys/TERMINATOR_KEY.asc | 28 ++++++++++++++++++++++ tests/integration/test_apt.py | 29 +++++++++++++---------- 3 files changed, 44 insertions(+), 42 deletions(-) delete mode 100644 tests/integration/keys/INKSCAPE_KEY.asc create mode 100644 tests/integration/keys/TERMINATOR_KEY.asc diff --git a/tests/integration/keys/INKSCAPE_KEY.asc b/tests/integration/keys/INKSCAPE_KEY.asc deleted file mode 100644 index 0c5637b5..00000000 --- a/tests/integration/keys/INKSCAPE_KEY.asc +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBGY0ViQBEACsQsdIRXzyEkk38x2oDt1yQ/Kt3dsiDKJFNLbs/xiDHrgIW6cU -1wZB0pfb3lCyG3/ZH5uvR0arCSHJvCvkdCFTqqkZndSA+/pXreCSLeP8CNawf/RM -3cNbdJlE8jXzaX2qzSEC9FDNqu4cQHIHR7xMAbSCPW8rxKvRCWmkZccfndDuQyN2 -vg3b2x9DWKS3DBRffivglF3yT49OuLemG5qJHujKOmNJZ32JoRniIivsuk1CCS1U -NDK6xWkr13aNe056QhVAh2iPF6MRE85zail+mQxt4LAgl/aLR0JSDSpWkbQH7kbu -5cJVan8nYF9HelVJ3QuMwdz3VQn4YVO2Wc8s0YfnNdQttYaUx3lz35Fct6CaOqjP -pgsZ4467lmB9ut74G+wDCQXmT232hsBkTz4D0DyVPB/ziBgGWUKti0AWNT/3kFww -2VM/80XADaDz0Sc/Hzi/cr9ZrbW3drMwoVvKtfMtOT7FMbeuPRWZKYZnDbvNm62e -ToKVudE0ijfsksxbcHKmGdvWntCxlinp0i39Jfz6y54pppjmbHRQgasmqm2EQLfA -RUNW/zB7gX80KTUCGjcLOTPajBIN5ZfjVITetryAFjv7fnK0qpf2OpqwF832W4V/ -S3GZtErupPktYG77Z9jOUxcJeEGYjWbVlbit8mTKDRdQUXOeOO6TzB4RcwARAQAB -tCVMYXVuY2hwYWQgUFBBIGZvciBJbmtzY2FwZSBEZXZlbG9wZXJziQJOBBMBCgA4 -FiEEVr3/0vHJaz0VdeO4XJoLhs0vyzgFAmY0ViQCGwMFCwkIBwIGFQoJCAsCBBYC -AwECHgECF4AACgkQXJoLhs0vyzh3RBAAo7Hee8i2I4n03/iq58lqml/OVJH9ZEle -amk3e0wsiVS0QdT/zB8/AMVDB1obazBfrHKJP9Ck+JKH0uxaGRxYBohTbO3Y3sBO -qRHz5VLcFzuyk7AA53AZkNx8Zbv6D0O4JTCPDJn9Gwbd/PpnvJm9Ri+zEiVPhXNu -oSBryGM09un2Yvi0DA+ulouSKTy9dkbI1R7flPZ2M/mKT8Lk0n1pJu5FvgPC4E6R -PT0Njw9+k/iHtP76U4SqHJZZx2I/TGlXMD1memyTK4adWZeGLaAiFadsoeJsDoDE -MkHFxFkr9n0E4wJhRGgL0OxDWugJemkUvHbzXFNUaeX5Spw/aO7r1CtTh8lyqiom -4ebAkURjESRFOFzcsM7nyQnmt2OgQkEShSL3NrDMkj1+3+FgQtd8sbeVpmpGBq87 -J3iq0YMsUysWq8DJSz4RnBTfeGlJWJkl3XxB9VbG3BqqbN9fRp+ccgZ51g5/DEA/ -q8jYS7Ur5bAlSoeA4M3SvKSlQM8+aT35dteFzejvo1N+2n0E0KoymuRsDBdzby0z -lJDKe244L5D6UPJo9YTmtE4kG/fGNZ5/JdRA+pbe7f/z84HVcJ3ziGdF/Nio/D8k -uFjZP2M/mxC7j+WnmKAruqmY+5vkAEqUPTobsloDjT3B+z0rzWk8FG/G5KFccsBO -2ekz6IVTXVA= -=VF33 ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/tests/integration/keys/TERMINATOR_KEY.asc b/tests/integration/keys/TERMINATOR_KEY.asc new file mode 100644 index 00000000..9e30ec0b --- /dev/null +++ b/tests/integration/keys/TERMINATOR_KEY.asc @@ -0,0 +1,28 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF+QbdoBEADVJllCeHgrQoKNH0ThjzarJAf379nA/y1Bxog9BsJCyyny/wlb +BJ9hUWdqTVgLAvrE4Dz4Y+ju0bjI4ukonzG4oTsFzncm20MmbcrmhdMZQXC2ONit +2+cRI05HGlgPlnIsMT0NRFJ3JjL0GSU+koSyU3SWb2Dgevnk2zkoo37xvsvF3DFx +gg1JvGhZyMiPDqnnBcknkdoQHIG/Qe8T0yCIz7Byu5Ej7WzPFznlHhg1ybmU0eze +TJOTo9R+57dD4hthi4329/E+6AuGZSZJBTRe2r1yW3tKpxx88b9RU6l/5fyJM6xi +udneOdcGKE6PNRCmKCAMXc5UGS4H1j5s5QcWdmXKtYG+b5tQcqX2DlCnCdAnmwMh +VlH1s9PI19leRkQnJs64cZkVBBqNhPQrBvvZpBLKXWB6YQBhRDf8FoP/zwdvKEZh +4JwFrRk6fnJ7ZmZFun58rMYnkTy8AX4wvvGPlfo2UgjDlEG9w82udIkTpCAgIOFG +4rQ4mcXvH6D7v7VhrIszKIDrtHTtV03FJntZCyAavbafyfx3OmMgWIZFXOxqk2Jm +hVm93CrZbrp3A7FnxcMa7mCuOw9j7kYe4ZmMj9eY3JiM+lrGB+WhA9mhmQc8kWWU +5wJphnmZ2SqEu/FD7B2UNQHHswRcTbGFBIjsgFE2FNFE6WFF7sB+uKeFLQARAQAB +tBpMYXVuY2hwYWQgUFBBIGZvciBNYXR0Um9zZYkCTgQTAQoAOBYhBGTdJhyFmKa6 +4qIL070v4KAeMWTbBQJfkG3aAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJ +EL0v4KAeMWTbrrMQAM5oNOWNSFmkCysxmd/geBT8B9koClVLWyiKeWdVtGvFjajq ++AUQ9xJpbQSf7wL1VkcqmL3HKAY6xwfT5qJCjvpljJVNMxpg12EMHBEzzFq5XnL7 +5l0ovf+LigexN0zU46Uc8vasDFDkRrR6Vftl4wq3DXKo70SPXX+rACN4R7Tt/jRE +D+XWKT2zJ1KZIt/8j064d9d2r6n6ozdMiO/WyR/oNQjqZHkqSxH7jOEfPp6u8PM6 +HnsyCBIWfliS5ieQ1hT3bp2Zl0qHSrvND4j+TnrmgU5LYhWhzvafEKyPoEkjWiIr +3zXRtqc40yVlYdMtu4LEesYxLUKt0Kx9yoAoe/UtT4feCFdXb/9S8CEqSfSDo8Hb +XVjKHjX0HL9os7xFX0f4Jeah7QHF0NiYPMqClDCe+/A49wvWz6JMaGhHy7qoa/hg +k8mmpSmWw24juMP6j3mMD0W8NfEOc0gXmFczJjGcm7WwYD6w3y1G8EOafM6CvzW6 +3M/sMt/NSKtXlXtflb1JJCcY6rLhXihjjE5HGZGvzTsNLWZi5kaIXOk/qBKf4gLJ +MjSgVI4iBvlsmQkQvGwuSJIkrXeNvzQIOyfllmA4L+i/8gGYJslxNtd19Ij/TMFs +glQXml9hi28VCApMCL64eF3sED9731RH7dYlW6xVFeNqL4Dg002y4ZTpXBPb +=3yD+ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index ec753d62..c9b004f1 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -91,37 +91,40 @@ def test_install_package_from_external_repository(): def test_install_higher_version_package_from_external_repository(): - repo_id = "deb-https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/-jammy" + repo_id = "deb-https://ppa.launchpadcontent.net/mattrose/terminator/ubuntu/-jammy" repos = apt.RepositoryMapping() + for r_id in repos._repository_map.keys(): + logger.info(r_id) assert repo_id not in repos ## version before - if not get_command_path("inkscape"): - apt.add_package("inkscape") + if not get_command_path("terminator"): + apt.add_package("terminator") + assert get_command_path("terminator") version_before = subprocess.run( - ["inkscape", "--version"], + ["terminator", "--version"], capture_output=True, check=True, text=True, ).stdout - apt.remove_package("inkscape") - assert not get_command_path("inkscape") + apt.remove_package("terminator") + assert not get_command_path("terminator") ## steps repo = apt.DebianRepository( enabled=True, repotype="deb", - uri="https://ppa.launchpadcontent.net/inkscape.dev/stable/ubuntu/", + uri="https://ppa.launchpadcontent.net/mattrose/terminator/ubuntu/", release="jammy", groups=["main"], ) repos.add(repo) # update_cache=False by default assert repo_id in repos assert repo_id in apt.RepositoryMapping() - key_file = apt.import_key((KEY_DIR / "INKSCAPE_KEY.asc").read_text()) + key_file = apt.import_key((KEY_DIR / "TERMINATOR_KEY.asc").read_text()) apt.update() - apt.add_package("inkscape") - assert get_command_path("inkscape") + apt.add_package("terminator") + assert get_command_path("terminator") version_after = subprocess.run( - ["inkscape", "--version"], + ["terminator", "--version"], capture_output=True, check=True, text=True, @@ -132,8 +135,8 @@ def test_install_higher_version_package_from_external_repository(): apt._add_repository(repo, remove=True) # pyright: ignore[reportPrivateUsage] assert repo_id not in apt.RepositoryMapping() apt.update() - apt.remove_package("inkscape") - assert not get_command_path("inkscape") + apt.remove_package("terminator") + assert not get_command_path("terminator") def test_install_hardware_observer_ssacli(): From 2b7cb4e6148b95c56e50d708450a1a49486ff56a Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 5 Dec 2024 15:31:11 +1300 Subject: [PATCH 48/50] test: switch from terminator to fish ppa for ci --- tests/integration/keys/FISH_KEY.asc | 29 ++++++++++++++++++++++ tests/integration/keys/TERMINATOR_KEY.asc | 28 --------------------- tests/integration/test_apt.py | 30 +++++++++++------------ 3 files changed, 43 insertions(+), 44 deletions(-) create mode 100644 tests/integration/keys/FISH_KEY.asc delete mode 100644 tests/integration/keys/TERMINATOR_KEY.asc diff --git a/tests/integration/keys/FISH_KEY.asc b/tests/integration/keys/FISH_KEY.asc new file mode 100644 index 00000000..edd2cf43 --- /dev/null +++ b/tests/integration/keys/FISH_KEY.asc @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGY0i8EBEAC787CEn0bb9R6lgxSjwEzrfUGK2HVfUoSfKiKILlU/p+nwdBdx +zKOGEtl2O8W4Q+KZkwEbD4llJKPFhzjF2CeP2qRKd/PLpiZoKhAr/NR7klQveqGx +Hop3uXsdGFxsNv2z0VEyJ7vhO+X0AuAW9BcXpXBexltrIzqbuGg+BKTwAOc5qaWx +xzDgcTvXR7xeudHv4vqXez93pd93WaRnjNB0vOUm03cKqFvfzfn1mJruThO3WE6S +ISjC/rQF4IquNS5ncA2+NLfb7fsPpYUmfKCTucYK2i929Xw/cwLfqE668kQLfJ6G +TiUK6B0EMK/Cvb/LYdN7RMXFrnWt6FnZORZM+9u4ZiCIa60OfW+xXjhmU6Lln3ZV +enhcD0qZHqZLQIxWGxK9rKJsasoBFnbDzBNyMM6BfukQqh6VP3UzdXixPCW1lOFl +omsPj3eoqmIf6tmyINMiJbmQrZmJ7Zn1tpTqScfdPJU06z26hcTQrn2+rvHtQV5V +ebj9JMqvKuk5mG7zkjFgMHoqP6jLlVqsReCfY66VuYLg4o2m0r2F+wv7ZfFXYcCq +LOycszdQe/SgJHobJc/yt/brZMrqFmGtYAERU7NcbRceuc4i2k1VnlygBT1AuvGT +HoSgm9ct0MZlfKRLhkZm+izwxLqxVSAsB/DZJ3uWp4XsKx4XHjw2VFw5aQARAQAB +tChMYXVuY2hwYWQgUFBBIGZvciBGaXNoIHNoZWxsIG1haW50YWluZXJziQJOBBMB +CgA4FiEEiEIecD7cevVJZ97Uc8n8yeK7SNoFAmY0i8ECGwMFCwkIBwIGFQoJCAsC +BBYCAwECHgECF4AACgkQc8n8yeK7SNoJUA//bnOSCrdit87tvsDmaok9+kQ2YYqL +X4KVsKxI2QVsKhRXPJb3yRwQRLArEUhT8yg5kQBIaHKdDhAkH9PRFZeCin25x/lq +LvjpnyslhlI9fhnImC21KqAPs8Svrb2LrvtfQK+072Fw5bISGJ6qlINU1vmJ0AIF +poojZaudcGLiQym1hkyf7HU4loHdkyUKX0jCTS8c9E8pRrDpwdHkKI3pEZpVWva1 +zsmPgYXR/RRCt4zV4/lwX2WiCmYBBGdXn3D2mfA+ONfExngDfjpzXAQ8Dp9m3bDI +AmtlaKRjHa9VKKd0wt6LH2JYmt52lUwoEvoiVf8M0poKrmPN+ft8EaOMpgORS8ut +RZzVndHx4jFyZ+pVDt00J16Bblq6MSaGZalPneIfycx4tuwPPTX7CALUeMQ0V/Gy +p053yvotLwsBC6KGjKugxUzgbSlp7WWiPEbYXZfjb7IZpNmKXtuvfyBo8rs3icez +ZkUmljl2J27bhSIUZQ124ham6yQ3B8MfXUdctiWUCkOdunqUuBeHa5yeKJHTcFUH +Y5nyyn79L5VLZ6YlwYn9qW0V/LvfnW2JmrXTVFjEhfUoEm0Zck8UBqWn2wfSagTI +2KWU87IgN3MF7QG5EiLEAUA/DfagBwgjCpRFH8VUeWqMjTkQs9kjB/kvCcvHGp8g +WVimNlxjmpCZsvk= +=5r0P +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/keys/TERMINATOR_KEY.asc b/tests/integration/keys/TERMINATOR_KEY.asc deleted file mode 100644 index 9e30ec0b..00000000 --- a/tests/integration/keys/TERMINATOR_KEY.asc +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBF+QbdoBEADVJllCeHgrQoKNH0ThjzarJAf379nA/y1Bxog9BsJCyyny/wlb -BJ9hUWdqTVgLAvrE4Dz4Y+ju0bjI4ukonzG4oTsFzncm20MmbcrmhdMZQXC2ONit -2+cRI05HGlgPlnIsMT0NRFJ3JjL0GSU+koSyU3SWb2Dgevnk2zkoo37xvsvF3DFx -gg1JvGhZyMiPDqnnBcknkdoQHIG/Qe8T0yCIz7Byu5Ej7WzPFznlHhg1ybmU0eze -TJOTo9R+57dD4hthi4329/E+6AuGZSZJBTRe2r1yW3tKpxx88b9RU6l/5fyJM6xi -udneOdcGKE6PNRCmKCAMXc5UGS4H1j5s5QcWdmXKtYG+b5tQcqX2DlCnCdAnmwMh -VlH1s9PI19leRkQnJs64cZkVBBqNhPQrBvvZpBLKXWB6YQBhRDf8FoP/zwdvKEZh -4JwFrRk6fnJ7ZmZFun58rMYnkTy8AX4wvvGPlfo2UgjDlEG9w82udIkTpCAgIOFG -4rQ4mcXvH6D7v7VhrIszKIDrtHTtV03FJntZCyAavbafyfx3OmMgWIZFXOxqk2Jm -hVm93CrZbrp3A7FnxcMa7mCuOw9j7kYe4ZmMj9eY3JiM+lrGB+WhA9mhmQc8kWWU -5wJphnmZ2SqEu/FD7B2UNQHHswRcTbGFBIjsgFE2FNFE6WFF7sB+uKeFLQARAQAB -tBpMYXVuY2hwYWQgUFBBIGZvciBNYXR0Um9zZYkCTgQTAQoAOBYhBGTdJhyFmKa6 -4qIL070v4KAeMWTbBQJfkG3aAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJ -EL0v4KAeMWTbrrMQAM5oNOWNSFmkCysxmd/geBT8B9koClVLWyiKeWdVtGvFjajq -+AUQ9xJpbQSf7wL1VkcqmL3HKAY6xwfT5qJCjvpljJVNMxpg12EMHBEzzFq5XnL7 -5l0ovf+LigexN0zU46Uc8vasDFDkRrR6Vftl4wq3DXKo70SPXX+rACN4R7Tt/jRE -D+XWKT2zJ1KZIt/8j064d9d2r6n6ozdMiO/WyR/oNQjqZHkqSxH7jOEfPp6u8PM6 -HnsyCBIWfliS5ieQ1hT3bp2Zl0qHSrvND4j+TnrmgU5LYhWhzvafEKyPoEkjWiIr -3zXRtqc40yVlYdMtu4LEesYxLUKt0Kx9yoAoe/UtT4feCFdXb/9S8CEqSfSDo8Hb -XVjKHjX0HL9os7xFX0f4Jeah7QHF0NiYPMqClDCe+/A49wvWz6JMaGhHy7qoa/hg -k8mmpSmWw24juMP6j3mMD0W8NfEOc0gXmFczJjGcm7WwYD6w3y1G8EOafM6CvzW6 -3M/sMt/NSKtXlXtflb1JJCcY6rLhXihjjE5HGZGvzTsNLWZi5kaIXOk/qBKf4gLJ -MjSgVI4iBvlsmQkQvGwuSJIkrXeNvzQIOyfllmA4L+i/8gGYJslxNtd19Ij/TMFs -glQXml9hi28VCApMCL64eF3sED9731RH7dYlW6xVFeNqL4Dg002y4ZTpXBPb -=3yD+ ------END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/test_apt.py b/tests/integration/test_apt.py index c9b004f1..5c61bd91 100644 --- a/tests/integration/test_apt.py +++ b/tests/integration/test_apt.py @@ -91,40 +91,38 @@ def test_install_package_from_external_repository(): def test_install_higher_version_package_from_external_repository(): - repo_id = "deb-https://ppa.launchpadcontent.net/mattrose/terminator/ubuntu/-jammy" + repo_id = "deb-https://ppa.launchpadcontent.net/fish-shell/release-3/ubuntu/-jammy" repos = apt.RepositoryMapping() - for r_id in repos._repository_map.keys(): - logger.info(r_id) assert repo_id not in repos ## version before - if not get_command_path("terminator"): - apt.add_package("terminator") - assert get_command_path("terminator") + if not get_command_path("fish"): + apt.add_package("fish") + assert get_command_path("fish") version_before = subprocess.run( - ["terminator", "--version"], + ["fish", "--version"], capture_output=True, check=True, text=True, ).stdout - apt.remove_package("terminator") - assert not get_command_path("terminator") + apt.remove_package("fish") + assert not get_command_path("fish") ## steps repo = apt.DebianRepository( enabled=True, repotype="deb", - uri="https://ppa.launchpadcontent.net/mattrose/terminator/ubuntu/", + uri="https://ppa.launchpadcontent.net/fish-shell/release-3/ubuntu/", release="jammy", groups=["main"], ) repos.add(repo) # update_cache=False by default assert repo_id in repos assert repo_id in apt.RepositoryMapping() - key_file = apt.import_key((KEY_DIR / "TERMINATOR_KEY.asc").read_text()) + key_file = apt.import_key((KEY_DIR / "FISH_KEY.asc").read_text()) apt.update() - apt.add_package("terminator") - assert get_command_path("terminator") + apt.add_package("fish") + assert get_command_path("fish") version_after = subprocess.run( - ["terminator", "--version"], + ["fish", "--version"], capture_output=True, check=True, text=True, @@ -135,8 +133,8 @@ def test_install_higher_version_package_from_external_repository(): apt._add_repository(repo, remove=True) # pyright: ignore[reportPrivateUsage] assert repo_id not in apt.RepositoryMapping() apt.update() - apt.remove_package("terminator") - assert not get_command_path("terminator") + apt.remove_package("fish") + assert not get_command_path("fish") def test_install_hardware_observer_ssacli(): From 709e2c755aedff28666954d68ad046cb680b4531 Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 5 Dec 2024 15:59:26 +1300 Subject: [PATCH 49/50] chore: remove ValueError from RepositoryMapping.add --- lib/charms/operator_libs_linux/v0/apt.py | 18 ++++++++++++++---- tests/unit/test_deb822.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py index bf2fe98a..1c67d400 100644 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -1428,14 +1428,16 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und """Add a new repository to the system using add-apt-repository. Args: - repo: a DebianRepository object where repo.enabled is True - raises: - ValueError: if repo.enabled is False + repo: a DebianRepository object + if repo.enabled is falsey, will return without adding the repository + Raises: CalledProcessError: if there's an error running apt-add-repository WARNING: Does not associate the repository with a signing key. Use `import_key` to add a signing key globally. + WARNING: if repo.enabled is falsey, will return without adding the repository + WARNING: Don't forget to call `apt.update` before installing any packages! Or call `apt.add_package` with `update_cache=True`. @@ -1443,7 +1445,15 @@ def add( # noqa: D417 # undocumented-param: default_filename intentionally und only. It is not used, and was not used in the previous revision of this library. """ if not repo.enabled: - raise ValueError("{repo}.enabled is {value}".format(repo=repo, value=repo.enabled)) + logger.warning( + ( + "Returning from RepositoryMapping.add(repo=%s) without adding the repo" + " because repo.enabled is %s" + ), + repo, + repo.enabled, + ) + return _add_repository(repo) self._repository_map[_repo_to_identifier(repo)] = repo diff --git a/tests/unit/test_deb822.py b/tests/unit/test_deb822.py index df2142e7..96bf88bc 100644 --- a/tests/unit/test_deb822.py +++ b/tests/unit/test_deb822.py @@ -334,12 +334,15 @@ def test_add_with_deb822(repo_mapping: apt.RepositoryMapping): # we re-raise CalledProcessError after logging error = apt.CalledProcessError(1, "cmd") error.stdout = error.stderr = b"" - with patch.object(apt.logger, "error") as mock_logging_error: - with patch.object(apt.subprocess, "run", side_effect=error): + with patch.object(apt.subprocess, "run", side_effect=error): + with patch.object(apt.logger, "error") as mock_error: with pytest.raises(apt.CalledProcessError): repo_mapping.add(repo) - mock_logging_error.assert_called_once() + mock_error.assert_called_once() # call add with a disabled repository repo._enabled = False - with pytest.raises(ValueError): - repo_mapping.add(repo) + with patch.object(apt, "_add_repository") as mock_add: + with patch.object(apt.logger, "warning") as mock_warning: + repo_mapping.add(repo) + mock_add.assert_not_called() + mock_warning.assert_called_once() From 28204972670b0533c4ac068c293ecb42b176efdd Mon Sep 17 00:00:00 2001 From: James Garner Date: Thu, 5 Dec 2024 16:00:55 +1300 Subject: [PATCH 50/50] test: remove old tests from test_repo.py --- tests/unit/test_repo.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/tests/unit/test_repo.py b/tests/unit/test_repo.py index 164a9c83..7b7cec1a 100644 --- a/tests/unit/test_repo.py +++ b/tests/unit/test_repo.py @@ -1,9 +1,6 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. -import os - -import pytest from charms.operator_libs_linux.v0 import apt from pyfakefs.fake_filesystem_unittest import TestCase @@ -89,23 +86,6 @@ def test_can_disable_repositories(self): open(other.filename).readlines(), ) - @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") - def test_can_add_repositories(self): - r = apt.RepositoryMapping() - d = apt.DebianRepository( - True, - "deb", - "http://example.com", - "test", - ["group"], - "/etc/apt/sources.list.d/example-test.list", - ) - r.add(d, default_filename=False) - self.assertIn( - "{} {} {} {}\n".format(d.repotype, d.uri, d.release, " ".join(d.groups)), - open(d.filename).readlines(), - ) - def test_can_create_repo_from_repo_line(self): d = apt.DebianRepository.from_repo_line( "deb https://example.com/foo focal bar baz", @@ -118,23 +98,6 @@ def test_can_create_repo_from_repo_line(self): self.assertEqual(d.groups, ["bar", "baz"]) self.assertEqual(d.filename, "/etc/apt/sources.list.d/foo-focal.list") - @pytest.mark.skip("RepositoryMapping.add now calls apt-add-repository") - def test_valid_list_file(self): - line = "deb https://repo.example.org/fiz/baz focal/foo-bar/5.0 multiverse" - d = apt.DebianRepository.from_repo_line(line) - self.assertEqual(d.filename, "/etc/apt/sources.list.d/fiz-baz-focal-foo-bar-5.0.list") - - r = apt.RepositoryMapping() - d = apt.DebianRepository( - True, - "deb", - "https://repo.example.org/fiz/baz", - "focal/foo-bar/5.0", - ["multiverse"], - ) - r.add(d, default_filename=False) - assert os.path.exists("/etc/apt/sources.list.d/fiz-baz-focal-foo-bar-5.0.list") - def test_can_add_repositories_from_string_with_options(self): d = apt.DebianRepository.from_repo_line( "deb [signed-by=/foo/gpg.key arch=amd64] https://example.com/foo focal bar baz",