From 878ccf850ce6dcb7429c38cd9d7f85578ee763f7 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 8 Nov 2024 14:10:42 +0100 Subject: [PATCH 1/7] Converting pacman usage from syscalls to libalpm, to avoid using syscalls to interact with pacman --- archinstall/__init__.py | 3 +- archinstall/lib/pacman/__init__.py | 409 +++++++++++++++++++++++++++-- 2 files changed, 393 insertions(+), 19 deletions(-) diff --git a/archinstall/__init__.py b/archinstall/__init__.py index aec47685cf..9273f83d9e 100644 --- a/archinstall/__init__.py +++ b/archinstall/__init__.py @@ -98,8 +98,7 @@ def define_arguments() -> None: parser.print_help() exit(0) if os.getuid() != 0: - print(_("Archinstall requires root privileges to run. See --help for more.")) - exit(1) + warn(_("Archinstall might require root privileges to run. See --help for more.")) def parse_unspecified_argument_list(unknowns: list, multiple: bool = False, err: bool = False) -> dict: diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index 1ce11850fe..e026308fcf 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -1,8 +1,15 @@ -from pathlib import Path import time import re +import typing +import pathlib +import tempfile +import pyalpm +import pydantic +import platform +import traceback +import urllib.parse +import shutil from typing import TYPE_CHECKING, Any, Callable, Union -from shutil import copy2 from ..general import SysCommand from ..output import warn, error, info @@ -15,13 +22,387 @@ _: Any -class Pacman: +class PacmanServer(pydantic.BaseModel): + address :urllib.parse.ParseResult + + def geturl(self): + return self.address.geturl() + + +class PacmanTransaction: + def __init__(self, + session :pyalpm.Handle, + cascade=False, + nodeps=False, + force=True, + dbonly=False, + downloadonly=False, + nosave=False, + recurse=False, + recurseall=False, + unneeded=False, + alldeps=False, + allexplicit=False + ): + self.cascade = cascade + self.nodeps = nodeps + self.force = force + self.dbonly = dbonly + self.downloadonly = downloadonly + self.nosave = nosave + self.recurse = recurse + self.recurseall = recurseall + self.unneeded = unneeded + self.alldeps = alldeps + self.allexplicit = allexplicit + + self._session = session + self._transaction = None + + def __enter__(self): + try: + self._transaction = self._session.init_transaction( + cascade=self.cascade, + nodeps=self.nodeps, + force=self.force, + dbonly=self.dbonly, + downloadonly=self.downloadonly, + nosave=self.nosave, + recurse=self.recurse, + recurseall=self.recurseall, + unneeded=self.unneeded, + alldeps=self.alldeps, + allexplicit=self.allexplicit + ) + except pyalpm.error as error: + message, code, _ = error.args + + if code == 10: + raise PermissionError(f"Could not lock database {db.name}.db in {self.dbpath}") + + raise error + return self + + def __exit__(self, exit_type, exit_value, exit_tb) -> None: + if self._transaction: + try: + self._transaction.prepare() + self._transaction.commit() + except pyalpm.error as error: + message, code, _ = error.args + + if code == 28: + # Transaction was not prepared + pass + else: + traceback.print_exc() + self._transaction.release() + return False + self._transaction.release() + return True + + def __getattr__(self, key): + # Transparency function to route calls directly towards + # self._transaction rather than implementing add_pkg() for instance + # in this class. + if self._transaction: + return getattr(self._transaction, key, None) + + return None + - def __init__(self, target: Path, silent: bool = False): - self.synced = False +class Pacman: + def __init__(self, + config :pathlib.Path = pathlib.Path('/etc/pacman.conf'), + servers :typing.List[str] | typing.Dict[str, PacmanServer] | None = None, + dbpath :pathlib.Path | None = None, # pathlib.Path('/var/lib/pacman/') + cachedir :pathlib.Path | None = None, # pathlib.Path('/var/cache/pacman/pkg') + hooks :typing.List[pathlib.Path] | None = None, + repos :typing.List[str] | None = None, + # hooks = [ + # pathlib.Path('/usr/share/libalpm/hooks/'), + # pathlib.Path('/etc/pacman.d/hooks/') + # ], + logfile :pathlib.Path | None = None, # pathlib.Path('/var/log/pacman.log'), + gpgdir :pathlib.Path | None = None, # pathlib.Path('/etc/pacman.d/gnupg/'), + lock :pathlib.Path | None = None, + include_config_mirrors :bool = False, + temporary :bool = False, + silent :bool = False, + synced :float | None = None, + **kwargs + ): + self.config = config + self.servers = servers + self.dbpath = dbpath + self.cachedir = cachedir + self.hookdir = hooks + self.logfile = logfile + self.gpgdir = gpgdir + self.lock = lock + self.repos = repos + self.temporary = temporary self.silent = silent - self.target = target + self.synced = synced + + self._temporary_pacman_root = None + self._session = None + self._source_config_mirrors = True if servers is None or include_config_mirrors is True else False + + self._config = self.load_config() + + if self.repos is None: + # Get the activated repositories from the config + self.repos = list(set(self._config.keys()) - {'options'}) + + if isinstance(self.servers, list): + _mapped_to_repos = { + + } + + for repo in self.repos: + _mapped_to_repos[repo] = [ + PacmanServer(address=urllib.parse.urlparse(server)) for server in self.servers + ] + + self.servers = _mapped_to_repos + elif self.servers is None: + self.servers = { + repo: self._config[repo] + for repo in self.repos + } + + def load_config(self): + """ + Loads the given pacman.conf (usually /etc/pacman.conf) + and initiates not-None-values. + So if you want to use a temporary location make sure + to specify values first and then load the config to not ovveride them. + """ + + print(f"Loading pacman configuration {self.config}") + + config = {} + with self.config.open('r') as fh: + _section = None + for line in fh: + if len(line.strip()) == 0 or line.startswith('#'): + continue + + if line.startswith('[') and line.endswith(']\n'): + _section = line[1:-2] + continue + + config_item = line.strip() + + if _section not in config: + config[_section] = {} + + if _section.lower() == 'options': + if '=' in config_item: + # Key = Value pair + key, value = config_item.split('=', 1) + key = key.lower() + + config[_section][key] = value + else: + config[_section][key] = True + + elif _section.lower() != 'options': + repo = _section + if isinstance(config[_section], dict): + # Only the [options] section is "key: value" pairs + # the repo sections are only Server= entries. + config[_section] = [] + + if self._source_config_mirrors: + # if self.servers is None: + # self.servers = {} + if '=' in config_item: + key, value = config_item.split('=', 1) + key = key.strip().lower() + value = value.strip() + + if key.lower() == 'include': + value = pathlib.Path(value).expanduser().resolve().absolute() + if value.exists() is False: + raise PermissionError(f"Could not open mirror definitions for [{repo}] by including {value}") + + with value.open('r') as mirrors: + for mirror_line in mirrors: + if len(mirror_line.strip()) == 0 or mirror_line.startswith('#'): + continue + + if '=' in mirror_line: + _, url = mirror_line.split('=', 1) + url = url.strip() + url_obj = urllib.parse.urlparse(url) + + config[repo].append( + PacmanServer(address=url_obj) + ) + return config + + def __enter__(self) -> 'Pacman': + """ + Because transactions in pacman rhymes well with python's context managers. + We implement transactions via the `with Pacman() as session` context. + This allows us to do Pacman(temporary=True) for temporary sessions + or Pacman() to use system-wide operations. + """ + + # A lot of inspiration is taken from pycman: https://github.com/archlinux/pyalpm/blob/6a0b75dac7151dfa2ea28f368db22ade1775ee2b/pycman/action_sync.py#L184 + if self.temporary: + with tempfile.TemporaryDirectory(delete=False) as temporary_pacman_root: + # First we set a bunch of general configurations + # which load_config() will honor as long as they are not None + # (Anything we set here = is honored by load_config()) + # + # These general configs point to our temporary directory + # to not interferer with the system-wide pacman stuff + self.rootdir = temporary_pacman_root + self._temporary_pacman_root = pathlib.Path(temporary_pacman_root) + self.cachedir = self._temporary_pacman_root / 'cache' + self.dbpath = self._temporary_pacman_root / 'db' + self.logfile = self._temporary_pacman_root / 'pacman.log' + self.cachedir.mkdir(parents=True, exist_ok=True) + self.dbpath.mkdir(parents=True, exist_ok=True) + + if self.lock is None: + self.lock = self.dbpath / 'db.lck' + + for key, value in self._config['options'].items(): + if getattr(self, key, None) is None: + setattr(self, key, value) + + if getattr(self, 'rootdir', None) is None: + self.rootdir = '/' + if getattr(self, 'dbpath', None) is None: + self.dbpath = '/var/lib/pacman/' + + # Session is our libalpm handle with 2 databases + self._session = pyalpm.Handle(str(self.rootdir), str(self.dbpath)) + for repo in self.repos: + self._session.register_syncdb(repo, pyalpm.SIG_DATABASE_OPTIONAL) + + self._session.cachedirs = [ + str(self.cachedir) + ] + + if self.temporary: + self.update() + + return self + + def __exit__(self, exit_type, exit_value, exit_tb) -> None: + if self.temporary: + shutil.rmtree(self._temporary_pacman_root.expanduser().resolve().absolute()) + + return None + + def update(self): + # We update our temporary (fresh) database so that + # we ensure we rely on the latest information + for db in self._session.get_syncdbs(): + # Set up a transaction with some sane defaults + with PacmanTransaction(session=self._session) as _transaction: + # Populate the database with the servers + # listed in the configuration (we could manually override a list here) + db.servers = [ + server.geturl().replace('$repo', db.name).replace('$arch', platform.machine()) + for server in self.servers[db.name] + ] + + db.update(force=True) + + def install(self, *packages): + pyalpm_package_list = [] + missing_packages = [] + + for package in packages: + if not (results := self.search(package, exact=True)): + missing_packages.append(package) + continue + + pyalpm_package_list += results + + if missing_packages: + raise ValueError(f"Could not find package(s): {' '.join(missing_packages)}") + + with PacmanTransaction(session=self._session) as _transaction: + print(f"Installing packages: {pyalpm_package_list}") + [_transaction.add_pkg(pkg) for pkg in pyalpm_package_list] + + def search(self, *patterns, exact=True): + results = [] + queries = [] + + if exact: + for pattern in patterns: + if pattern.startswith('^') is False: + pattern = "^" + pattern + if pattern.endswith('$') is False: + pattern += '$' + queries.append(pattern) + else: + queries = patterns + + for db in self._session.get_syncdbs(): + # print(f"Searching {db.name} for: {' '.join(patterns)}") + results += db.search(*queries) + + # !! Workaround for https://gitlab.archlinux.org/pacman/pacman/-/issues/204 + # + # Since the regex ^$ should make absolute matches + # but doesn't. This could be because I assume (incorrectly) that + # `pacman -Ss ` should match on `pkgname` and `pkgdescr` in PKGBUILD + # or because it's a bug. + # But we can remove the following workaround once that is sorted out: + if exact: + results = [ + package + for package in results + if package.name in patterns or f"^{package.name}$" in patterns + ] + + return results + + def query(self, *patterns, exact=True): + results = [] + queries = [] + + if exact: + for pattern in patterns: + if pattern.startswith('^') is False: + pattern = "^" + pattern + if pattern.endswith('$') is False: + pattern += '$' + queries.append(pattern) + else: + queries = patterns + + db = self._session.get_localdb() + results += db.search(*queries) + + # !! Workaround for https://gitlab.archlinux.org/pacman/pacman/-/issues/204 + # + # Since the regex ^$ should make absolute matches + # but doesn't. This could be because I assume (incorrectly) that + # `pacman -Ss ` should match on `pkgname` and `pkgdescr` in PKGBUILD + # or because it's a bug. + # But we can remove the following workaround once that is sorted out: + if exact: + results = [ + package + for package in results + if package.name in patterns or f"^{package.name}$" in patterns + ] + + return results + + # These are the old functions, that have been retrofitted + # to work as before - but with the new pacman logic. @staticmethod def run(args: str, default_cmd: str = 'pacman') -> SysCommand: """ @@ -29,7 +410,7 @@ def run(args: str, default_cmd: str = 'pacman') -> SysCommand: It also protects us from colliding with other running pacman sessions (if used locally). The grace period is set to 10 minutes before exiting hard if another pacman instance is running. """ - pacman_db_lock = Path('/var/lib/pacman/db.lck') + pacman_db_lock = pathlib.Path('/var/lib/pacman/db.lck') if pacman_db_lock.exists(): warn(_('Pacman is already running, waiting maximum 10 minutes for it to terminate.')) @@ -58,16 +439,10 @@ def ask(self, error_message: str, bail_message: str, func: Callable, *args, **kw def sync(self) -> None: if self.synced: return - self.ask( - 'Could not sync a new package database', - 'Could not sync mirrors', - self.run, - '-Syy', - default_cmd='/usr/bin/pacman' - ) + self.update() self.synced = True - def strap(self, packages: Union[str, list[str]]) -> None: + def strap(self, target, packages: Union[str, list[str]]) -> None: self.sync() if isinstance(packages, str): packages = [packages] @@ -83,6 +458,6 @@ def strap(self, packages: Union[str, list[str]]) -> None: 'Could not strap in packages', 'Pacstrap failed. See /var/log/archinstall/install.log or above message for error details', SysCommand, - f'/usr/bin/pacstrap -C /etc/pacman.conf -K {self.target} {" ".join(packages)} --noconfirm', + f'/usr/bin/pacstrap -C /etc/pacman.conf -K {target} {" ".join(packages)} --noconfirm', peek_output=True - ) + ) \ No newline at end of file From 7853ff5ca4ee6dafe48a7bdc76f0cef6ba02b8b6 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 8 Nov 2024 14:44:52 +0100 Subject: [PATCH 2/7] Added pyalpm as a dependency and fixed flake8 --- PKGBUILD | 1 + archinstall/lib/pacman/__init__.py | 68 ++++++++++++++---------------- pyproject.toml | 3 +- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/PKGBUILD b/PKGBUILD index 7a8affeab3..bfc04feb24 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -22,6 +22,7 @@ depends=( 'pciutils' 'procps-ng' 'python' + 'pyalpm' 'python-pydantic' 'python-pyparted' 'python-simple-term-menu' diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index e026308fcf..1fdc1e5af2 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -23,15 +23,16 @@ class PacmanServer(pydantic.BaseModel): - address :urllib.parse.ParseResult + address: urllib.parse.ParseResult def geturl(self): return self.address.geturl() class PacmanTransaction: - def __init__(self, - session :pyalpm.Handle, + def __init__( + self, + session: pyalpm.Handle, cascade=False, nodeps=False, force=True, @@ -78,7 +79,7 @@ def __enter__(self): message, code, _ = error.args if code == 10: - raise PermissionError(f"Could not lock database {db.name}.db in {self.dbpath}") + raise PermissionError(f"Could not lock database {self._session.dbpath}") raise error return self @@ -112,24 +113,25 @@ def __getattr__(self, key): class Pacman: - def __init__(self, - config :pathlib.Path = pathlib.Path('/etc/pacman.conf'), - servers :typing.List[str] | typing.Dict[str, PacmanServer] | None = None, - dbpath :pathlib.Path | None = None, # pathlib.Path('/var/lib/pacman/') - cachedir :pathlib.Path | None = None, # pathlib.Path('/var/cache/pacman/pkg') - hooks :typing.List[pathlib.Path] | None = None, - repos :typing.List[str] | None = None, + def __init__( + self, + config: pathlib.Path = pathlib.Path('/etc/pacman.conf'), + servers: typing.List[str] | typing.Dict[str, PacmanServer] | None = None, + dbpath: pathlib.Path | None = None, # pathlib.Path('/var/lib/pacman/') + cachedir: pathlib.Path | None = None, # pathlib.Path('/var/cache/pacman/pkg') + hooks: typing.List[pathlib.Path] | None = None, + repos: typing.List[str] | None = None, # hooks = [ # pathlib.Path('/usr/share/libalpm/hooks/'), # pathlib.Path('/etc/pacman.d/hooks/') # ], - logfile :pathlib.Path | None = None, # pathlib.Path('/var/log/pacman.log'), - gpgdir :pathlib.Path | None = None, # pathlib.Path('/etc/pacman.d/gnupg/'), - lock :pathlib.Path | None = None, - include_config_mirrors :bool = False, - temporary :bool = False, - silent :bool = False, - synced :float | None = None, + logfile: pathlib.Path | None = None, # pathlib.Path('/var/log/pacman.log'), + gpgdir: pathlib.Path | None = None, # pathlib.Path('/etc/pacman.d/gnupg/'), + lock: pathlib.Path | None = None, + include_config_mirrors: bool = False, + temporary: bool = False, + silent: bool = False, + synced: float | None = None, **kwargs ): self.config = config @@ -194,7 +196,7 @@ def load_config(self): continue config_item = line.strip() - + if _section not in config: config[_section] = {} @@ -306,7 +308,7 @@ def update(self): # we ensure we rely on the latest information for db in self._session.get_syncdbs(): # Set up a transaction with some sane defaults - with PacmanTransaction(session=self._session) as _transaction: + with PacmanTransaction(session=self._session) as _transaction: # noqa: F841 # Populate the database with the servers # listed in the configuration (we could manually override a list here) db.servers = [ @@ -326,7 +328,7 @@ def install(self, *packages): continue pyalpm_package_list += results - + if missing_packages: raise ValueError(f"Could not find package(s): {' '.join(missing_packages)}") @@ -352,13 +354,10 @@ def search(self, *patterns, exact=True): # print(f"Searching {db.name} for: {' '.join(patterns)}") results += db.search(*queries) - # !! Workaround for https://gitlab.archlinux.org/pacman/pacman/-/issues/204 - # - # Since the regex ^$ should make absolute matches - # but doesn't. This could be because I assume (incorrectly) that - # `pacman -Ss ` should match on `pkgname` and `pkgdescr` in PKGBUILD - # or because it's a bug. - # But we can remove the following workaround once that is sorted out: + # Because `pacman -Ss ` doesn't perform exact matches, + # as discussed in https://gitlab.archlinux.org/pacman/pacman/-/issues/204 + # we need to filter out any inexact results until we figure out if libalpm + # can be used to implement `pacman --sync --nodeps --nodeps --print --print-format '%n' nano` if exact: results = [ package @@ -385,13 +384,10 @@ def query(self, *patterns, exact=True): db = self._session.get_localdb() results += db.search(*queries) - # !! Workaround for https://gitlab.archlinux.org/pacman/pacman/-/issues/204 - # - # Since the regex ^$ should make absolute matches - # but doesn't. This could be because I assume (incorrectly) that - # `pacman -Ss ` should match on `pkgname` and `pkgdescr` in PKGBUILD - # or because it's a bug. - # But we can remove the following workaround once that is sorted out: + # Because `pacman -Ss ` doesn't perform exact matches, + # as discussed in https://gitlab.archlinux.org/pacman/pacman/-/issues/204 + # we need to filter out any inexact results until we figure out if libalpm + # can be used to implement `pacman --sync --nodeps --nodeps --print --print-format '%n' nano` if exact: results = [ package @@ -460,4 +456,4 @@ def strap(self, target, packages: Union[str, list[str]]) -> None: SysCommand, f'/usr/bin/pacstrap -C /etc/pacman.conf -K {target} {" ".join(packages)} --noconfirm', peek_output=True - ) \ No newline at end of file + ) diff --git a/pyproject.toml b/pyproject.toml index f304fcd93c..e6a471d588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,8 @@ classifiers = [ dependencies = [ "simple-term-menu==1.6.4", "pyparted @ https://github.com//dcantrell/pyparted/archive/v3.13.0.tar.gz#sha512=26819e28d73420937874f52fda03eb50ab1b136574ea9867a69d46ae4976d38c4f26a2697fa70597eed90dd78a5ea209bafcc3227a17a7a5d63cff6d107c2b11", - "pydantic==2.9.2" + "pydantic==2.9.2", + "pyalpm==0.10.6", ] [project.urls] From 088d5ca374379ed2158918ab453fb3a3ae3bbd06 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 8 Nov 2024 15:45:58 +0100 Subject: [PATCH 3/7] Ignored pyalpm in pylint, as it struggles with the C-wrapper --- .github/workflows/pylint.yaml | 2 +- archinstall/lib/installer.py | 24 ++++++++++++------------ archinstall/lib/pacman/__init__.py | 12 ++++++------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml index 23dfa0132a..aa5390e05a 100644 --- a/.github/workflows/pylint.yaml +++ b/.github/workflows/pylint.yaml @@ -19,4 +19,4 @@ jobs: - run: python --version - run: pylint --version - name: Lint with Pylint - run: pylint . + run: pylint --extension-pkg-allow-list=pyalpm . diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index ccd59de043..c5fa70da0c 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -92,7 +92,7 @@ def __init__( self._zram_enabled = False self._disable_fstrim = False - self.pacman = Pacman(self.target, storage['arguments'].get('silent', False)) + self.pacman = Pacman(silent=storage['arguments'].get('silent', False)) def __enter__(self) -> 'Installer': return self @@ -672,7 +672,7 @@ def post_install_enable_iwd_service(*args: str, **kwargs: str) -> None: # Otherwise, we can go ahead and add the required package # and enable it's service: else: - self.pacman.strap('iwd') + self.pacman.strap(self.target, 'iwd') self.enable_service('iwd') for psk in psk_files: @@ -768,7 +768,7 @@ def _handle_partition_installation(self) -> None: if part in self._disk_encryption.partitions: if self._disk_encryption.hsm_device: # Required by mkinitcpio to add support for fido2-device options - self.pacman.strap('libfido2') + self.pacman.strap(self.target, 'libfido2') if 'sd-encrypt' not in self._hooks: self._hooks.insert(self._hooks.index('filesystems'), 'sd-encrypt') @@ -804,7 +804,7 @@ def _handle_lvm_installation(self) -> None: if self._disk_encryption.encryption_type in [disk.EncryptionType.LvmOnLuks, disk.EncryptionType.LuksOnLvm]: if self._disk_encryption.hsm_device: # Required by mkinitcpio to add support for fido2-device options - self.pacman.strap('libfido2') + self.pacman.strap(self.target, 'libfido2') if 'sd-encrypt' not in self._hooks: self._hooks.insert(self._hooks.index('lvm2'), 'sd-encrypt') @@ -851,7 +851,7 @@ def minimal_installation( pacman_conf.apply() - self.pacman.strap(self._base_packages) + self.pacman.strap(self.target, self._base_packages) self.helper_flags['base-strapped'] = True pacman_conf.persist() @@ -894,7 +894,7 @@ def minimal_installation( def setup_swap(self, kind: str = 'zram') -> None: if kind == 'zram': info("Setting up swap on zram") - self.pacman.strap('zram-generator') + self.pacman.strap(self.target, 'zram-generator') # We could use the default example below, but maybe not the best idea: https://github.com/archlinux/archinstall/pull/678#issuecomment-962124813 # zram_example_location = '/usr/share/doc/zram-generator/zram-generator.conf.example' @@ -1056,7 +1056,7 @@ def _add_systemd_bootloader( ) -> None: debug('Installing systemd bootloader') - self.pacman.strap('efibootmgr') + self.pacman.strap(self.target, 'efibootmgr') if not SysInfo.has_uefi(): raise HardwareIncompatibilityError @@ -1154,7 +1154,7 @@ def _add_grub_bootloader( ) -> None: debug('Installing grub bootloader') - self.pacman.strap('grub') # no need? + self.pacman.strap(self.target, 'grub') # no need? grub_default = self.target / 'etc/default/grub' config = grub_default.read_text() @@ -1181,7 +1181,7 @@ def _add_grub_bootloader( info(f"GRUB EFI partition: {efi_partition.dev_path}") - self.pacman.strap('efibootmgr') # TODO: Do we need? Yes, but remove from minimal_installation() instead? + self.pacman.strap(self.target, 'efibootmgr') # TODO: Do we need? Yes, but remove from minimal_installation() instead? boot_dir_arg = [] if boot_partition.mountpoint and boot_partition.mountpoint != boot_dir: @@ -1239,7 +1239,7 @@ def _add_limine_bootloader( ) -> None: debug('Installing limine bootloader') - self.pacman.strap('limine') + self.pacman.strap(self.target, 'limine') info(f"Limine boot partition: {boot_partition.dev_path}") @@ -1329,7 +1329,7 @@ def _add_efistub_bootloader( ) -> None: debug('Installing efistub bootloader') - self.pacman.strap('efibootmgr') + self.pacman.strap(self.target, 'efibootmgr') if not SysInfo.has_uefi(): raise HardwareIncompatibilityError @@ -1466,7 +1466,7 @@ def add_bootloader(self, bootloader: Bootloader, uki_enabled: bool = False): self._add_limine_bootloader(boot_partition, efi_partition, root) def add_additional_packages(self, packages: Union[str, List[str]]) -> None: - return self.pacman.strap(packages) + return self.pacman.strap(self.target, packages) def enable_sudo(self, entity: str, group: bool = False): info(f'Enabling sudo permissions for {entity}') diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index 1fdc1e5af2..d138a217d1 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -76,7 +76,7 @@ def __enter__(self): allexplicit=self.allexplicit ) except pyalpm.error as error: - message, code, _ = error.args + message, code, _ = error.args # pylint: disable=unbalanced-tuple-unpacking if code == 10: raise PermissionError(f"Could not lock database {self._session.dbpath}") @@ -90,7 +90,7 @@ def __exit__(self, exit_type, exit_value, exit_tb) -> None: self._transaction.prepare() self._transaction.commit() except pyalpm.error as error: - message, code, _ = error.args + message, code, _ = error.args # pylint: disable=unbalanced-tuple-unpacking if code == 28: # Transaction was not prepared @@ -318,7 +318,7 @@ def update(self): db.update(force=True) - def install(self, *packages): + def install(self, *packages: typing.List[str]): pyalpm_package_list = [] missing_packages = [] @@ -336,7 +336,7 @@ def install(self, *packages): print(f"Installing packages: {pyalpm_package_list}") [_transaction.add_pkg(pkg) for pkg in pyalpm_package_list] - def search(self, *patterns, exact=True): + def search(self, *patterns: typing.List[str], exact: bool = True): results = [] queries = [] @@ -367,7 +367,7 @@ def search(self, *patterns, exact=True): return results - def query(self, *patterns, exact=True): + def query(self, *patterns: typing.List[str], exact: bool = True): results = [] queries = [] @@ -438,7 +438,7 @@ def sync(self) -> None: self.update() self.synced = True - def strap(self, target, packages: Union[str, list[str]]) -> None: + def strap(self, target: str, packages: Union[str, list[str]]) -> None: self.sync() if isinstance(packages, str): packages = [packages] From f6d280342525db2d417c5c35f7c8afcccf4fb09c Mon Sep 17 00:00:00 2001 From: Torxed Date: Fri, 8 Nov 2024 18:40:47 +0100 Subject: [PATCH 4/7] Added pyalpm to the dependencies, as well as fixed all mypy issues --- archinstall/lib/exceptions.py | 3 ++ archinstall/lib/pacman/__init__.py | 77 ++++++++++++++++++------------ pyproject.toml | 1 + 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/archinstall/lib/exceptions.py b/archinstall/lib/exceptions.py index f11308deca..b9a9289654 100644 --- a/archinstall/lib/exceptions.py +++ b/archinstall/lib/exceptions.py @@ -44,3 +44,6 @@ class DownloadTimeout(Exception): ''' Download timeout exception raised by DownloadTimer. ''' + +class PacmanIssue(Exception): + pass \ No newline at end of file diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index d138a217d1..655592a2ac 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -15,7 +15,7 @@ from ..output import warn, error, info from .repo import Repo from .config import Config -from ..exceptions import RequirementError +from ..exceptions import RequirementError, PacmanIssue from ..plugins import plugins if TYPE_CHECKING: @@ -100,7 +100,6 @@ def __exit__(self, exit_type, exit_value, exit_tb) -> None: self._transaction.release() return False self._transaction.release() - return True def __getattr__(self, key): # Transparency function to route calls directly towards @@ -116,7 +115,7 @@ class Pacman: def __init__( self, config: pathlib.Path = pathlib.Path('/etc/pacman.conf'), - servers: typing.List[str] | typing.Dict[str, PacmanServer] | None = None, + servers: typing.List[str] | typing.Dict[str, typing.List[PacmanServer]] | None = None, dbpath: pathlib.Path | None = None, # pathlib.Path('/var/lib/pacman/') cachedir: pathlib.Path | None = None, # pathlib.Path('/var/cache/pacman/pkg') hooks: typing.List[pathlib.Path] | None = None, @@ -132,6 +131,7 @@ def __init__( temporary: bool = False, silent: bool = False, synced: float | None = None, + rootdir: pathlib.Path | None = None, **kwargs ): self.config = config @@ -146,8 +146,9 @@ def __init__( self.temporary = temporary self.silent = silent self.synced = synced + self.rootdir = rootdir - self._temporary_pacman_root = None + self._temporary_pacman_root :pathlib.Path | None = None self._session = None self._source_config_mirrors = True if servers is None or include_config_mirrors is True else False @@ -155,9 +156,10 @@ def __init__( if self.repos is None: # Get the activated repositories from the config - self.repos = list(set(self._config.keys()) - {'options'}) + # and if none are set, use None + self.repos = list(set(self._config.keys()) - {'options'}) or None - if isinstance(self.servers, list): + if self.repos is not None and isinstance(self.servers, list): _mapped_to_repos = { } @@ -168,13 +170,13 @@ def __init__( ] self.servers = _mapped_to_repos - elif self.servers is None: + elif self.repos and self.servers is None: self.servers = { repo: self._config[repo] for repo in self.repos } - def load_config(self): + def load_config(self) -> typing.Dict[str, Any]: """ Loads the given pacman.conf (usually /etc/pacman.conf) and initiates not-None-values. @@ -184,7 +186,7 @@ def load_config(self): print(f"Loading pacman configuration {self.config}") - config = {} + config :typing.Dict[str, typing.Any] = {} with self.config.open('r') as fh: _section = None for line in fh: @@ -197,10 +199,10 @@ def load_config(self): config_item = line.strip() - if _section not in config: - config[_section] = {} + if _section and _section not in config: + config[_section] = {} - if _section.lower() == 'options': + if _section and _section.lower() == 'options': if '=' in config_item: # Key = Value pair key, value = config_item.split('=', 1) @@ -210,7 +212,7 @@ def load_config(self): else: config[_section][key] = True - elif _section.lower() != 'options': + elif _section and _section.lower() != 'options': repo = _section if isinstance(config[_section], dict): # Only the [options] section is "key: value" pairs @@ -227,11 +229,11 @@ def load_config(self): value = value.strip() if key.lower() == 'include': - value = pathlib.Path(value).expanduser().resolve().absolute() - if value.exists() is False: - raise PermissionError(f"Could not open mirror definitions for [{repo}] by including {value}") + absolute_path = pathlib.Path(value).expanduser().resolve().absolute() + if absolute_path.exists() is False: + raise PermissionError(f"Could not open mirror definitions for [{repo}] by including {absolute_path}") - with value.open('r') as mirrors: + with absolute_path.open('r') as mirrors: for mirror_line in mirrors: if len(mirror_line.strip()) == 0 or mirror_line.startswith('#'): continue @@ -256,7 +258,7 @@ def __enter__(self) -> 'Pacman': # A lot of inspiration is taken from pycman: https://github.com/archlinux/pyalpm/blob/6a0b75dac7151dfa2ea28f368db22ade1775ee2b/pycman/action_sync.py#L184 if self.temporary: - with tempfile.TemporaryDirectory(delete=False) as temporary_pacman_root: + with tempfile.TemporaryDirectory(delete=False) as temporary_pacman_root: # type: ignore # First we set a bunch of general configurations # which load_config() will honor as long as they are not None # (Anything we set here = is honored by load_config()) @@ -279,12 +281,18 @@ def __enter__(self) -> 'Pacman': setattr(self, key, value) if getattr(self, 'rootdir', None) is None: - self.rootdir = '/' + self.rootdir = pathlib.Path('/') if getattr(self, 'dbpath', None) is None: - self.dbpath = '/var/lib/pacman/' + self.dbpath = pathlib.Path('/var/lib/pacman/') + if self.repos is None: + raise PacmanIssue(f"No repositories are configured.") # Session is our libalpm handle with 2 databases self._session = pyalpm.Handle(str(self.rootdir), str(self.dbpath)) + + if not self._session: + raise PacmanIssue(f"Could not initate a pyalpm handle.") + for repo in self.repos: self._session.register_syncdb(repo, pyalpm.SIG_DATABASE_OPTIONAL) @@ -298,12 +306,15 @@ def __enter__(self) -> 'Pacman': return self def __exit__(self, exit_type, exit_value, exit_tb) -> None: - if self.temporary: + if self.temporary and self._temporary_pacman_root: shutil.rmtree(self._temporary_pacman_root.expanduser().resolve().absolute()) return None - def update(self): + def update(self) -> None: + if not self._session: + raise PacmanIssue("Pacman() needs to be executed in a context") + # We update our temporary (fresh) database so that # we ensure we rely on the latest information for db in self._session.get_syncdbs(): @@ -318,7 +329,7 @@ def update(self): db.update(force=True) - def install(self, *packages: typing.List[str]): + def install(self, *packages: str): pyalpm_package_list = [] missing_packages = [] @@ -336,9 +347,12 @@ def install(self, *packages: typing.List[str]): print(f"Installing packages: {pyalpm_package_list}") [_transaction.add_pkg(pkg) for pkg in pyalpm_package_list] - def search(self, *patterns: typing.List[str], exact: bool = True): - results = [] - queries = [] + def search(self, *patterns: str, exact: bool = True): + results :typing.List[pyalpm.Package] = [] + queries :typing.List[str] = [] + + if not self._session: + raise PacmanIssue("Pacman() needs to be executed in a context") if exact: for pattern in patterns: @@ -367,9 +381,12 @@ def search(self, *patterns: typing.List[str], exact: bool = True): return results - def query(self, *patterns: typing.List[str], exact: bool = True): - results = [] - queries = [] + def query(self, *patterns: str, exact: bool = True): + results :typing.List[pyalpm.Package] = [] + queries :typing.List[str] = [] + + if not self._session: + raise PacmanIssue("Pacman() needs to be executed in a context") if exact: for pattern in patterns: @@ -438,7 +455,7 @@ def sync(self) -> None: self.update() self.synced = True - def strap(self, target: str, packages: Union[str, list[str]]) -> None: + def strap(self, target: pathlib.Path, packages: Union[str, list[str]]) -> None: self.sync() if isinstance(packages, str): packages = [packages] diff --git a/pyproject.toml b/pyproject.toml index e6a471d588..c000076107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ warn_unreachable = false module = [ "parted", "simple_term_menu", + "pyalpm" ] ignore_missing_imports = true From 2a4ffd489895b75bd23041f54eedcbdd08ac248c Mon Sep 17 00:00:00 2001 From: Torxed Date: Fri, 8 Nov 2024 18:42:27 +0100 Subject: [PATCH 5/7] Fixed ruff complaints --- archinstall/lib/exceptions.py | 2 +- archinstall/lib/pacman/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/archinstall/lib/exceptions.py b/archinstall/lib/exceptions.py index b9a9289654..8d93c308c1 100644 --- a/archinstall/lib/exceptions.py +++ b/archinstall/lib/exceptions.py @@ -46,4 +46,4 @@ class DownloadTimeout(Exception): ''' class PacmanIssue(Exception): - pass \ No newline at end of file + pass diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index 655592a2ac..c5862ed72b 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -285,13 +285,13 @@ def __enter__(self) -> 'Pacman': if getattr(self, 'dbpath', None) is None: self.dbpath = pathlib.Path('/var/lib/pacman/') if self.repos is None: - raise PacmanIssue(f"No repositories are configured.") + raise PacmanIssue("No repositories are configured.") # Session is our libalpm handle with 2 databases self._session = pyalpm.Handle(str(self.rootdir), str(self.dbpath)) if not self._session: - raise PacmanIssue(f"Could not initate a pyalpm handle.") + raise PacmanIssue("Could not initate a pyalpm handle.") for repo in self.repos: self._session.register_syncdb(repo, pyalpm.SIG_DATABASE_OPTIONAL) From c1111b127a4484e975c963ae87c4d63b4b504187 Mon Sep 17 00:00:00 2001 From: Torxed Date: Fri, 8 Nov 2024 18:44:58 +0100 Subject: [PATCH 6/7] Fixed flake8 --- archinstall/lib/exceptions.py | 1 + archinstall/lib/pacman/__init__.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/archinstall/lib/exceptions.py b/archinstall/lib/exceptions.py index 8d93c308c1..2c2e8d4db3 100644 --- a/archinstall/lib/exceptions.py +++ b/archinstall/lib/exceptions.py @@ -45,5 +45,6 @@ class DownloadTimeout(Exception): Download timeout exception raised by DownloadTimer. ''' + class PacmanIssue(Exception): pass diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index c5862ed72b..cb7d0c1463 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -148,7 +148,7 @@ def __init__( self.synced = synced self.rootdir = rootdir - self._temporary_pacman_root :pathlib.Path | None = None + self._temporary_pacman_root: pathlib.Path | None = None self._session = None self._source_config_mirrors = True if servers is None or include_config_mirrors is True else False @@ -186,7 +186,7 @@ def load_config(self) -> typing.Dict[str, Any]: print(f"Loading pacman configuration {self.config}") - config :typing.Dict[str, typing.Any] = {} + config: typing.Dict[str, typing.Any] = {} with self.config.open('r') as fh: _section = None for line in fh: @@ -200,7 +200,7 @@ def load_config(self) -> typing.Dict[str, Any]: config_item = line.strip() if _section and _section not in config: - config[_section] = {} + config[_section] = {} if _section and _section.lower() == 'options': if '=' in config_item: @@ -348,8 +348,8 @@ def install(self, *packages: str): [_transaction.add_pkg(pkg) for pkg in pyalpm_package_list] def search(self, *patterns: str, exact: bool = True): - results :typing.List[pyalpm.Package] = [] - queries :typing.List[str] = [] + results: typing.List[pyalpm.Package] = [] + queries: typing.List[str] = [] if not self._session: raise PacmanIssue("Pacman() needs to be executed in a context") @@ -382,8 +382,8 @@ def search(self, *patterns: str, exact: bool = True): return results def query(self, *patterns: str, exact: bool = True): - results :typing.List[pyalpm.Package] = [] - queries :typing.List[str] = [] + results: typing.List[pyalpm.Package] = [] + queries: typing.List[str] = [] if not self._session: raise PacmanIssue("Pacman() needs to be executed in a context") From e364ec40bce14ea19eb4a31991af17aead5c16c4 Mon Sep 17 00:00:00 2001 From: Torxed Date: Fri, 8 Nov 2024 22:13:14 +0100 Subject: [PATCH 7/7] Added pyalpm arch package to build workflow --- .github/workflows/python-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-build.yml b/.github/workflows/python-build.yml index 5909961458..a4d8da5972 100644 --- a/.github/workflows/python-build.yml +++ b/.github/workflows/python-build.yml @@ -17,7 +17,7 @@ jobs: pacman-key --init pacman --noconfirm -Sy archlinux-keyring pacman --noconfirm -Syyu - pacman --noconfirm -Sy python-pip python-pydantic python-pyparted python-simple-term-menu pkgconfig gcc + pacman --noconfirm -Sy pyalpm python-pip python-pydantic python-pyparted python-simple-term-menu pkgconfig gcc - name: Install build dependencies run: | python -m pip install --break-system-packages --upgrade pip