diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a2a42687b1..5bb4b61c55 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -56,7 +56,7 @@ jobs: python-version: '3.10' - name: Run benchmarks - run: pytest --benchmark-only --benchmark-json benchmark.json + run: pytest --db-backend psql --benchmark-only --benchmark-json benchmark.json tests/ - name: Store benchmark result uses: aiidateam/github-action-benchmark@v3 @@ -73,4 +73,4 @@ jobs: alert-threshold: 200% comment-on-alert: true fail-on-alert: false - alert-comment-cc-users: '@chrisjsewell,@giovannipizzi' + alert-comment-cc-users: '@giovannipizzi,@agoscinski,@GeigerJ2,@khsrali,@unkcpz' diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index f49853733e..57e85bb004 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -104,7 +104,7 @@ jobs: AIIDA_WARN_v3: 1 # Python 3.12 has a performance regression when running with code coverage # so run code coverage only for python 3.9. - run: pytest -v tests -n auto -m 'not nightly' ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }} + run: pytest -n auto --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }} - name: Upload coverage report if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core' @@ -139,7 +139,7 @@ jobs: - name: Run test suite env: AIIDA_WARN_v3: 0 - run: pytest -n auto -m 'presto' + run: pytest -n auto -m 'presto' tests/ verdi: diff --git a/.github/workflows/ci-style.yml b/.github/workflows/ci-style.yml deleted file mode 100644 index 1f4b549ad2..0000000000 --- a/.github/workflows/ci-style.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: ci-style - -on: - push: - branches-ignore: [gh-pages] - pull_request: - branches-ignore: [gh-pages] - -env: - FORCE_COLOR: 1 - -jobs: - - pre-commit: - - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v4 - - - name: Install python dependencies - uses: ./.github/actions/install-aiida-core - with: - python-version: '3.11' - extras: '[pre-commit]' - from-requirements: 'false' - - - name: Run pre-commit - run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c7ea8cb787..84ed617125 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -104,19 +104,6 @@ jobs: rabbitmq-version: ['3.11', '3.12', '3.13'] services: - postgres: - image: postgres:16 - env: - POSTGRES_DB: test_aiida - POSTGRES_PASSWORD: '' - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 rabbitmq: image: rabbitmq:${{ matrix.rabbitmq-version }}-management ports: @@ -132,9 +119,6 @@ jobs: with: python-version: '3.11' - - name: Install system dependencies - run: sudo apt update && sudo apt install postgresql - - name: Setup SSH on localhost run: .github/workflows/setup_ssh.sh @@ -145,7 +129,7 @@ jobs: id: tests env: AIIDA_WARN_v3: 0 - run: pytest -sv -m 'requires_rmq' + run: pytest -s --db-backend sqlite -m 'requires_rmq' tests/ - name: Slack notification # Always run this step (otherwise it would be skipped if any of the previous steps fail) but only if the diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80f7e35326..c47595baee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,19 +54,6 @@ jobs: timeout-minutes: 30 services: - postgres: - image: postgres:10 - env: - POSTGRES_DB: test_aiida - POSTGRES_PASSWORD: '' - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 rabbitmq: image: rabbitmq:3.8.14-management ports: @@ -76,16 +63,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt update - sudo apt install postgresql graphviz - - name: Install aiida-core uses: ./.github/actions/install-aiida-core - name: Run sub-set of test suite - run: pytest -sv -k 'requires_rmq' + run: pytest -s -m requires_rmq --db-backend=sqlite tests/ publish: diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 2f919a8c99..d315a50691 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -229,7 +229,7 @@ jobs: env: AIIDA_TEST_PROFILE: test_aiida AIIDA_WARN_v3: 1 - run: pytest --verbose tests -m 'not nightly' + run: pytest --db-backend psql tests -m 'not nightly' tests/ - name: Freeze test environment run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt diff --git a/.github/workflows/tests_nightly.sh b/.github/workflows/tests_nightly.sh index 10f26f7f15..2712a5124e 100755 --- a/.github/workflows/tests_nightly.sh +++ b/.github/workflows/tests_nightly.sh @@ -13,4 +13,4 @@ verdi -p test_aiida run ${SYSTEM_TESTS}/test_containerized_code.py bash ${SYSTEM_TESTS}/test_polish_workchains.sh verdi daemon stop -AIIDA_TEST_PROFILE=test_aiida pytest -v tests -m 'nightly' +AIIDA_TEST_PROFILE=test_aiida pytest --db-backend psql -m nightly tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32305828b4..7a88a2ab99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,6 +90,7 @@ repos: .docker/.*| docs/.*| utils/.*| + tests/.*| src/aiida/calculations/arithmetic/add.py| src/aiida/calculations/diff_tutorial/calculations.py| @@ -124,7 +125,6 @@ repos: src/aiida/engine/processes/ports.py| src/aiida/manage/configuration/__init__.py| src/aiida/manage/configuration/config.py| - src/aiida/manage/configuration/profile.py| src/aiida/manage/external/rmq/launcher.py| src/aiida/manage/tests/main.py| src/aiida/manage/tests/pytest_fixtures.py| @@ -192,17 +192,6 @@ repos: src/aiida/transports/plugins/local.py| src/aiida/transports/plugins/ssh.py| src/aiida/workflows/arithmetic/multiply_add.py| - - tests/conftest.py| - tests/repository/conftest.py| - tests/repository/test_repository.py| - tests/sphinxext/sources/workchain/conf.py| - tests/sphinxext/sources/workchain_broken/conf.py| - tests/storage/psql_dos/migrations/conftest.py| - tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py| - tests/test_calculation_node.py| - tests/test_nodes.py| - )$ - id: dm-generate-all diff --git a/CHANGELOG.md b/CHANGELOG.md index ea3c843d8a..04327d7105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v2.6.3 - 2024-11-6 + +### Fixes +- CLI: Fix exception for `verdi plugin list` (#6560) [[c3b10b7]](https://github.com/aiidateam/aiida-core/commit/c3b10b759a9cd062800ef120591d5c7fd0ae4ee7) +- `DirectScheduler`: Ensure killing child processes (#6572) [[fddffca]](https://github.com/aiidateam/aiida-core/commit/fddffca67b4f7e3b76b19df7db8e1511c449d2d9) +- Engine: Fix state change broadcast before process node is updated (#6580) [[867353c]](https://github.com/aiidateam/aiida-core/commit/867353c415c61d94a2427d5225dd5224a1b95fb9) + +### Devops +- Docker: Replace sleep with `s6-notifyoncheck` (#6475) [[9579378b]](https://github.com/aiidateam/aiida-core/commit/9579378ba063237baa5b73380eb8e9f0a28529ee) +- Fix failed docker CI using more reasoning grep regex to parse python version (#6581) [[332a4a91]](https://github.com/aiidateam/aiida-core/commit/332a4a915771afedcb144463b012558e4669e529) +- DevOps: Fix json query in reading the docker names to filter out fields not starting with aiida (#6573) [[e1467edc]](https://github.com/aiidateam/aiida-core/commit/e1467edca902867e53605e0e60b67f8767bf8d3e) + + ## v2.6.2 - 2024-08-07 ### Fixes @@ -31,7 +44,7 @@ ## v2.6.1 - 2024-07-01 -### Fixes: +### Fixes - Fixtures: Make `pgtest` truly an optional dependency [[9fe8fd2e0]](https://github.com/aiidateam/aiida-core/commit/9fe8fd2e0b88e746ee2156eccb71b7adbab6b2c5) diff --git a/pyproject.toml b/pyproject.toml index 4c652a8738..231863037f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,7 +211,7 @@ notebook = [ ] pre-commit = [ 'aiida-core[atomic_tools,rest,tests,tui]', - 'mypy~=1.10.0', + 'mypy~=1.13.0', 'packaging~=23.0', 'pre-commit~=3.5', 'sqlalchemy[mypy]~=2.0', @@ -256,6 +256,7 @@ runaiida = 'aiida.cmdline.commands.cmd_run:run' verdi = 'aiida.cmdline.commands.cmd_verdi:verdi' [project.urls] +Changelog = 'https://github.com/aiidateam/aiida-core/blob/main/CHANGELOG.md' Documentation = 'https://aiida.readthedocs.io' Home = 'http://www.aiida.net/' Source = 'https://github.com/aiidateam/aiida-core' diff --git a/src/aiida/cmdline/commands/cmd_presto.py b/src/aiida/cmdline/commands/cmd_presto.py index 09d7070e7c..eeb98fad75 100644 --- a/src/aiida/cmdline/commands/cmd_presto.py +++ b/src/aiida/cmdline/commands/cmd_presto.py @@ -67,7 +67,7 @@ def detect_postgres_config( """ import secrets - from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER + from aiida.manage.configuration.settings import AiiDAConfigDir from aiida.manage.external.postgres import Postgres dbinfo = { @@ -92,13 +92,15 @@ def detect_postgres_config( except Exception as exception: raise ConnectionError(f'Unable to automatically create the PostgreSQL user and database: {exception}') + aiida_config_folder = AiiDAConfigDir.get() + return { 'database_hostname': postgres_hostname, 'database_port': postgres_port, 'database_name': database_name, 'database_username': database_username, 'database_password': database_password, - 'repository_uri': f'file://{AIIDA_CONFIG_FOLDER / "repository" / profile_name}', + 'repository_uri': f'file://{aiida_config_folder / "repository" / profile_name}', } diff --git a/src/aiida/cmdline/commands/cmd_profile.py b/src/aiida/cmdline/commands/cmd_profile.py index 3dd21b56bf..7cb0e018ae 100644 --- a/src/aiida/cmdline/commands/cmd_profile.py +++ b/src/aiida/cmdline/commands/cmd_profile.py @@ -169,9 +169,9 @@ def profile_list(): # This can happen for a fresh install and the `verdi setup` has not yet been run. In this case it is still nice # to be able to see the configuration directory, for instance for those who have set `AIIDA_PATH`. This way # they can at least verify that it is correctly set. - from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER + from aiida.manage.configuration.settings import AiiDAConfigDir - echo.echo_report(f'configuration folder: {AIIDA_CONFIG_FOLDER}') + echo.echo_report(f'configuration folder: {AiiDAConfigDir.get()}') echo.echo_critical(str(exception)) else: echo.echo_report(f'configuration folder: {config.dirpath}') diff --git a/src/aiida/cmdline/commands/cmd_status.py b/src/aiida/cmdline/commands/cmd_status.py index f3c32327dc..85ef292fa7 100644 --- a/src/aiida/cmdline/commands/cmd_status.py +++ b/src/aiida/cmdline/commands/cmd_status.py @@ -61,13 +61,14 @@ def verdi_status(print_traceback, no_rmq): from aiida.common.docs import URL_NO_BROKER from aiida.common.exceptions import ConfigurationError from aiida.engine.daemon.client import DaemonException, DaemonNotRunningException - from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER + from aiida.manage.configuration.settings import AiiDAConfigDir from aiida.manage.manager import get_manager exit_code = ExitCode.SUCCESS + configure_directory = AiiDAConfigDir.get() print_status(ServiceStatus.UP, 'version', f'AiiDA v{__version__}') - print_status(ServiceStatus.UP, 'config', AIIDA_CONFIG_FOLDER) + print_status(ServiceStatus.UP, 'config', configure_directory) manager = get_manager() diff --git a/src/aiida/cmdline/params/options/commands/setup.py b/src/aiida/cmdline/params/options/commands/setup.py index 930aa97018..49cfc1e121 100644 --- a/src/aiida/cmdline/params/options/commands/setup.py +++ b/src/aiida/cmdline/params/options/commands/setup.py @@ -66,11 +66,12 @@ def get_repository_uri_default(ctx): """ import os - from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER + from aiida.manage.configuration.settings import AiiDAConfigDir validate_profile_parameter(ctx) + configure_directory = AiiDAConfigDir.get() - return os.path.join(AIIDA_CONFIG_FOLDER, 'repository', ctx.params['profile'].name) + return os.path.join(configure_directory, 'repository', ctx.params['profile'].name) def get_quicksetup_repository_uri(ctx, param, value): diff --git a/src/aiida/cmdline/utils/ascii_vis.py b/src/aiida/cmdline/utils/ascii_vis.py index f42317e7a8..502abf3bcf 100644 --- a/src/aiida/cmdline/utils/ascii_vis.py +++ b/src/aiida/cmdline/utils/ascii_vis.py @@ -29,7 +29,7 @@ def calc_info(node, call_link_label: bool = False) -> str: raise TypeError(f'Unknown type: {type(node)}') process_label = node.process_label - process_state = node.process_state.value.capitalize() + process_state = 'None' if node.process_state is None else node.process_state.value.capitalize() exit_status = node.exit_status if call_link_label and (caller := node.caller): diff --git a/src/aiida/common/lang.py b/src/aiida/common/lang.py index ec9fb45ddb..6a8f4d3d5b 100644 --- a/src/aiida/common/lang.py +++ b/src/aiida/common/lang.py @@ -96,4 +96,4 @@ def __init__(self, getter: Callable[[SelfType], ReturnType]) -> None: self.getter = getter def __get__(self, instance: Any, owner: SelfType) -> ReturnType: - return self.getter(owner) + return self.getter(owner) # type: ignore[arg-type] diff --git a/src/aiida/common/pydantic.py b/src/aiida/common/pydantic.py index 633e83428d..2fa3d2bd68 100644 --- a/src/aiida/common/pydantic.py +++ b/src/aiida/common/pydantic.py @@ -41,7 +41,7 @@ class Model(BaseModel): field_info = Field(default, **kwargs) for key, value in (('priority', priority), ('short_name', short_name), ('option_cls', option_cls)): - if value is not None: + if value is not None and field_info is not None: field_info.metadata.append({key: value}) return field_info diff --git a/src/aiida/engine/daemon/client.py b/src/aiida/engine/daemon/client.py index ef250802f7..4e47e7ed60 100644 --- a/src/aiida/engine/daemon/client.py +++ b/src/aiida/engine/daemon/client.py @@ -94,10 +94,10 @@ def __init__(self, profile: Profile): from aiida.common.docs import URL_NO_BROKER type_check(profile, Profile) - config = get_config() + self._config = get_config() self._profile = profile self._socket_directory: str | None = None - self._daemon_timeout: int = config.get_option('daemon.timeout', scope=profile.name) + self._daemon_timeout: int = self._config.get_option('daemon.timeout', scope=profile.name) if self._profile.process_control_backend is None: raise ConfigurationError( @@ -156,31 +156,31 @@ def virtualenv(self) -> str | None: @property def circus_log_file(self) -> str: - return self.profile.filepaths['circus']['log'] + return self._config.filepaths(self.profile)['circus']['log'] @property def circus_pid_file(self) -> str: - return self.profile.filepaths['circus']['pid'] + return self._config.filepaths(self.profile)['circus']['pid'] @property def circus_port_file(self) -> str: - return self.profile.filepaths['circus']['port'] + return self._config.filepaths(self.profile)['circus']['port'] @property def circus_socket_file(self) -> str: - return self.profile.filepaths['circus']['socket']['file'] + return self._config.filepaths(self.profile)['circus']['socket']['file'] @property def circus_socket_endpoints(self) -> dict[str, str]: - return self.profile.filepaths['circus']['socket'] + return self._config.filepaths(self.profile)['circus']['socket'] @property def daemon_log_file(self) -> str: - return self.profile.filepaths['daemon']['log'] + return self._config.filepaths(self.profile)['daemon']['log'] @property def daemon_pid_file(self) -> str: - return self.profile.filepaths['daemon']['pid'] + return self._config.filepaths(self.profile)['daemon']['pid'] def get_circus_port(self) -> int: """Retrieve the port for the circus controller, which should be written to the circus port file. diff --git a/src/aiida/engine/processes/calcjobs/calcjob.py b/src/aiida/engine/processes/calcjobs/calcjob.py index d5acfca5fc..8ced783a5f 100644 --- a/src/aiida/engine/processes/calcjobs/calcjob.py +++ b/src/aiida/engine/processes/calcjobs/calcjob.py @@ -1062,7 +1062,7 @@ def presubmit(self, folder: Folder) -> CalcInfo: def encoder(obj): if dataclasses.is_dataclass(obj): - return dataclasses.asdict(obj) + return dataclasses.asdict(obj) # type: ignore[arg-type] raise TypeError(f' {obj!r} is not JSON serializable') subfolder = folder.get_subfolder('.aiida', create=True) diff --git a/src/aiida/engine/processes/workchains/restart.py b/src/aiida/engine/processes/workchains/restart.py index ad5cd8a181..34544704f2 100644 --- a/src/aiida/engine/processes/workchains/restart.py +++ b/src/aiida/engine/processes/workchains/restart.py @@ -29,7 +29,7 @@ def validate_handler_overrides( - process_class: 'BaseRestartWorkChain', handler_overrides: Optional[orm.Dict], ctx: 'PortNamespace' + process_class: type['BaseRestartWorkChain'], handler_overrides: Optional[orm.Dict], ctx: 'PortNamespace' ) -> Optional[str]: """Validator for the ``handler_overrides`` input port of the ``BaseRestartWorkChain``. diff --git a/src/aiida/manage/configuration/__init__.py b/src/aiida/manage/configuration/__init__.py index 7227281507..097fcfc012 100644 --- a/src/aiida/manage/configuration/__init__.py +++ b/src/aiida/manage/configuration/__init__.py @@ -65,10 +65,10 @@ def get_config_path(): - """Returns path to .aiida configuration directory.""" - from .settings import AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME + """Returns path to aiida configuration file.""" + from .settings import DEFAULT_CONFIG_FILE_NAME, AiiDAConfigDir - return os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME) + return os.path.join(AiiDAConfigDir.get(), DEFAULT_CONFIG_FILE_NAME) def load_config(create=False) -> 'Config': diff --git a/src/aiida/manage/configuration/config.py b/src/aiida/manage/configuration/config.py index 4b1f032271..fff83f4330 100644 --- a/src/aiida/manage/configuration/config.py +++ b/src/aiida/manage/configuration/config.py @@ -21,6 +21,7 @@ import json import os import uuid +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from pydantic import ( @@ -780,3 +781,32 @@ def _atomic_write(self, filepath=None): handle.flush() os.rename(handle.name, self.filepath) + + def filepaths(self, profile: Profile): + """Return the filepaths used by a profile. + + :return: a dictionary of filepaths + """ + from aiida.manage.configuration.settings import AiiDAConfigPathResolver + + _config_path_resolver: AiiDAConfigPathResolver = AiiDAConfigPathResolver(Path(self.dirpath)) + daemon_dir = _config_path_resolver.daemon_dir + daemon_log_dir = _config_path_resolver.daemon_log_dir + + return { + 'circus': { + 'log': str(daemon_log_dir / f'circus-{profile.name}.log'), + 'pid': str(daemon_dir / f'circus-{profile.name}.pid'), + 'port': str(daemon_dir / f'circus-{profile.name}.port'), + 'socket': { + 'file': str(daemon_dir / f'circus-{profile.name}.sockets'), + 'controller': 'circus.c.sock', + 'pubsub': 'circus.p.sock', + 'stats': 'circus.s.sock', + }, + }, + 'daemon': { + 'log': str(daemon_log_dir / f'aiida-{profile.name}.log'), + 'pid': str(daemon_dir / f'aiida-{profile.name}.pid'), + }, + } diff --git a/src/aiida/manage/configuration/profile.py b/src/aiida/manage/configuration/profile.py index acaca2e892..93a3c5c911 100644 --- a/src/aiida/manage/configuration/profile.py +++ b/src/aiida/manage/configuration/profile.py @@ -56,7 +56,7 @@ def __init__(self, name: str, config: Mapping[str, Any], validate=True): ) self._name = name - self._attributes: Dict[str, Any] = deepcopy(config) + self._attributes: Dict[str, Any] = deepcopy(config) # type: ignore[arg-type] # Create a default UUID if not specified if self._attributes.get(self.KEY_UUID, None) is None: @@ -235,22 +235,28 @@ def filepaths(self): :return: a dictionary of filepaths """ - from .settings import DAEMON_DIR, DAEMON_LOG_DIR + from aiida.common.warnings import warn_deprecation + from aiida.manage.configuration.settings import AiiDAConfigPathResolver + + warn_deprecation('This method has been deprecated, use `filepaths` method from `Config` obj instead', version=3) + + daemon_dir = AiiDAConfigPathResolver().daemon_dir + daemon_log_dir = AiiDAConfigPathResolver().daemon_log_dir return { 'circus': { - 'log': str(DAEMON_LOG_DIR / f'circus-{self.name}.log'), - 'pid': str(DAEMON_DIR / f'circus-{self.name}.pid'), - 'port': str(DAEMON_DIR / f'circus-{self.name}.port'), + 'log': str(daemon_log_dir / f'circus-{self.name}.log'), + 'pid': str(daemon_dir / f'circus-{self.name}.pid'), + 'port': str(daemon_dir / f'circus-{self.name}.port'), 'socket': { - 'file': str(DAEMON_DIR / f'circus-{self.name}.sockets'), + 'file': str(daemon_dir / f'circus-{self.name}.sockets'), 'controller': 'circus.c.sock', 'pubsub': 'circus.p.sock', 'stats': 'circus.s.sock', }, }, 'daemon': { - 'log': str(DAEMON_LOG_DIR / f'aiida-{self.name}.log'), - 'pid': str(DAEMON_DIR / f'aiida-{self.name}.pid'), + 'log': str(daemon_log_dir / f'aiida-{self.name}.log'), + 'pid': str(daemon_dir / f'aiida-{self.name}.pid'), }, } diff --git a/src/aiida/manage/configuration/settings.py b/src/aiida/manage/configuration/settings.py index 168abb8879..f47a2dd66e 100644 --- a/src/aiida/manage/configuration/settings.py +++ b/src/aiida/manage/configuration/settings.py @@ -13,6 +13,7 @@ import os import pathlib import warnings +from typing import final DEFAULT_UMASK = 0o0077 DEFAULT_AIIDA_PATH_VARIABLE = 'AIIDA_PATH' @@ -25,38 +26,86 @@ DEFAULT_DAEMON_LOG_DIR_NAME = 'log' DEFAULT_ACCESS_CONTROL_DIR_NAME = 'access' -# Assign defaults which may be overriden in set_configuration_directory() below -AIIDA_CONFIG_FOLDER: pathlib.Path = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME -DAEMON_DIR: pathlib.Path = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME -DAEMON_LOG_DIR: pathlib.Path = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME -ACCESS_CONTROL_DIR: pathlib.Path = AIIDA_CONFIG_FOLDER / DEFAULT_ACCESS_CONTROL_DIR_NAME +__all__ = ('AiiDAConfigPathResolver', 'AiiDAConfigDir') -def create_instance_directories() -> None: +@final +class AiiDAConfigDir: + """Singleton for setting and getting the path to configuration directory.""" + + _glb_aiida_config_folder: pathlib.Path = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME + + @classmethod + def get(cls): + """Return the path of the configuration directory.""" + return cls._glb_aiida_config_folder + + @classmethod + def set(cls, aiida_config_folder: pathlib.Path | None = None) -> None: + """Set the configuration directory, related global variables and create instance directories. + + The location of the configuration directory is defined by ``aiida_config_folder`` or if not defined, + the path that is returned by ``_get_configuration_directory_from_envvar``. + Or if the environment_variable not set, use the default. + If the directory does not exist yet, it is created, together with all its subdirectories. + """ + _default_dirpath_config = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME + + aiida_config_folder = aiida_config_folder or _get_configuration_directory_from_envvar() + cls._glb_aiida_config_folder = aiida_config_folder or _default_dirpath_config + + _create_instance_directories(cls._glb_aiida_config_folder) + + +@final +class AiiDAConfigPathResolver: + """For resolving configuration directory, daemon dir, daemon log dir and access control dir. + The locations are all trivially derived from the config directory. + """ + + def __init__(self, config_folder: pathlib.Path | None = None) -> None: + self._aiida_path = config_folder or AiiDAConfigDir.get() + + @property + def aiida_path(self) -> pathlib.Path: + return self._aiida_path + + @property + def daemon_dir(self) -> pathlib.Path: + return self._aiida_path / DEFAULT_DAEMON_DIR_NAME + + @property + def daemon_log_dir(self) -> pathlib.Path: + return self._aiida_path / DEFAULT_DAEMON_DIR_NAME / DEFAULT_DAEMON_LOG_DIR_NAME + + @property + def access_control_dir(self) -> pathlib.Path: + return self._aiida_path / DEFAULT_ACCESS_CONTROL_DIR_NAME + + +def _create_instance_directories(aiida_config_folder: pathlib.Path | None) -> None: """Create the base directories required for a new AiiDA instance. - This will create the base AiiDA directory defined by the AIIDA_CONFIG_FOLDER variable, unless it already exists. - Subsequently, it will create the daemon directory within it and the daemon log directory. + This will create the base AiiDA directory in ``aiida_config_folder``. + If it not provided, the directory returned from ``AiiDAConfigDir.get()`` will be the default config folder, + unless it already exists. Subsequently, it will create the daemon directory within it and the daemon log directory. """ from aiida.common import ConfigurationError - directory_base = AIIDA_CONFIG_FOLDER.expanduser() - directory_daemon = directory_base / DAEMON_DIR - directory_daemon_log = directory_base / DAEMON_LOG_DIR - directory_access = directory_base / ACCESS_CONTROL_DIR + path_resolver = AiiDAConfigPathResolver(aiida_config_folder) list_of_paths = [ - directory_base, - directory_daemon, - directory_daemon_log, - directory_access, + path_resolver.aiida_path, + path_resolver.daemon_dir, + path_resolver.daemon_log_dir, + path_resolver.access_control_dir, ] umask = os.umask(DEFAULT_UMASK) try: for path in list_of_paths: - if path is directory_base and not path.exists(): + if path is path_resolver.aiida_path and not path.exists(): warnings.warn(f'Creating AiiDA configuration folder `{path}`.') try: @@ -64,31 +113,10 @@ def create_instance_directories() -> None: except OSError as exc: raise ConfigurationError(f'could not create the `{path}` configuration directory: {exc}') from exc finally: - os.umask(umask) + _ = os.umask(umask) -def get_configuration_directory(): - """Return the path of the configuration directory. - - The location of the configuration directory is defined following these heuristics in order: - - * If the ``AIIDA_PATH`` variable is set, all the paths will be checked to see if they contain a - configuration folder. The first one to be encountered will be set as ``AIIDA_CONFIG_FOLDER``. If none of them - contain one, the last path defined in the environment variable considered is used. - * If an existing directory is still not found, the ``DEFAULT_AIIDA_PATH`` is used. - - :returns: The path of the configuration directory. - """ - dirpath_config = get_configuration_directory_from_envvar() - - # If no existing configuration directory is found, fall back to the default - if dirpath_config is None: - dirpath_config = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME - - return dirpath_config - - -def get_configuration_directory_from_envvar() -> pathlib.Path | None: +def _get_configuration_directory_from_envvar() -> pathlib.Path | None: """Return the path of a config directory from the ``AIIDA_PATH`` environment variable. The environment variable should be a colon separated string of filepaths that either point directly to a config @@ -99,10 +127,13 @@ def get_configuration_directory_from_envvar() -> pathlib.Path | None: """ environment_variable = os.environ.get(DEFAULT_AIIDA_PATH_VARIABLE) + default_dirpath_config = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME + if environment_variable is None: return None # Loop over all the paths in the ``AIIDA_PATH`` variable to see if any of them contain a configuration folder + dirpath_config = None for base_dir_path in [path for path in environment_variable.split(':') if path]: dirpath_config = pathlib.Path(base_dir_path).expanduser() @@ -115,28 +146,8 @@ def get_configuration_directory_from_envvar() -> pathlib.Path | None: if dirpath_config.is_dir(): break - return dirpath_config - - -def set_configuration_directory(aiida_config_folder: pathlib.Path | None = None) -> None: - """Set the configuration directory, related global variables and create instance directories. - - The location of the configuration directory is defined by ``aiida_config_folder`` or if not defined, the path that - is returned by ``get_configuration_directory``. If the directory does not exist yet, it is created, together with - all its subdirectories. - """ - global AIIDA_CONFIG_FOLDER # noqa: PLW0603 - global DAEMON_DIR # noqa: PLW0603 - global DAEMON_LOG_DIR # noqa: PLW0603 - global ACCESS_CONTROL_DIR # noqa: PLW0603 - - AIIDA_CONFIG_FOLDER = aiida_config_folder or get_configuration_directory() - DAEMON_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME - DAEMON_LOG_DIR = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME - ACCESS_CONTROL_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_ACCESS_CONTROL_DIR_NAME - - create_instance_directories() + return dirpath_config or default_dirpath_config # Initialize the configuration directory settings -set_configuration_directory() +AiiDAConfigDir.set() diff --git a/src/aiida/manage/manager.py b/src/aiida/manage/manager.py index 8621b324f4..651190454e 100644 --- a/src/aiida/manage/manager.py +++ b/src/aiida/manage/manager.py @@ -430,7 +430,7 @@ def create_runner(self, with_persistence: bool = True, **kwargs: Any) -> 'Runner if with_persistence and 'persister' not in settings: settings['persister'] = self.get_persister() - return runners.Runner(**settings) + return runners.Runner(**settings) # type: ignore[arg-type] def create_daemon_runner(self, loop: Optional['asyncio.AbstractEventLoop'] = None) -> 'Runner': """Create and return a new daemon runner. diff --git a/src/aiida/manage/profile_access.py b/src/aiida/manage/profile_access.py index 5b04481e66..c65364af03 100644 --- a/src/aiida/manage/profile_access.py +++ b/src/aiida/manage/profile_access.py @@ -18,8 +18,10 @@ from aiida.common.exceptions import LockedProfileError, LockingProfileError from aiida.common.lang import type_check from aiida.manage.configuration import Profile +from aiida.manage.configuration.settings import AiiDAConfigPathResolver +@typing.final class ProfileAccessManager: """Class to manage access to a profile. @@ -45,12 +47,10 @@ def __init__(self, profile: Profile): :param profile: the profile whose access to manage. """ - from aiida.manage.configuration.settings import ACCESS_CONTROL_DIR - - type_check(profile, Profile) + _ = type_check(profile, Profile) self.profile = profile self.process = psutil.Process(os.getpid()) - self._dirpath_records = ACCESS_CONTROL_DIR / profile.name + self._dirpath_records = AiiDAConfigPathResolver().access_control_dir / profile.name self._dirpath_records.mkdir(exist_ok=True) def request_access(self) -> None: diff --git a/src/aiida/manage/tests/pytest_fixtures.py b/src/aiida/manage/tests/pytest_fixtures.py index 92856aff66..6c5d04d3bc 100644 --- a/src/aiida/manage/tests/pytest_fixtures.py +++ b/src/aiida/manage/tests/pytest_fixtures.py @@ -162,6 +162,7 @@ def aiida_instance( """ from aiida.manage import configuration from aiida.manage.configuration import settings + from aiida.manage.configuration.settings import AiiDAConfigDir if aiida_test_profile: yield configuration.get_config() @@ -178,8 +179,7 @@ def aiida_instance( dirpath_config = tmp_path_factory.mktemp('config') os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = str(dirpath_config) - settings.AIIDA_CONFIG_FOLDER = dirpath_config - settings.set_configuration_directory() + AiiDAConfigDir.set(dirpath_config) configuration.CONFIG = configuration.load_config(create=True) try: @@ -191,7 +191,7 @@ def aiida_instance( else: os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = current_path_variable - settings.AIIDA_CONFIG_FOLDER = current_config_path + AiiDAConfigDir.set(current_config_path) configuration.CONFIG = current_config if current_profile: aiida_manager.load_profile(current_profile.name, allow_switch=True) diff --git a/src/aiida/orm/nodes/data/array/projection.py b/src/aiida/orm/nodes/data/array/projection.py index e58442d9c5..881ac727c2 100644 --- a/src/aiida/orm/nodes/data/array/projection.py +++ b/src/aiida/orm/nodes/data/array/projection.py @@ -278,7 +278,7 @@ def array_list_checker(array_list, array_name, orb_length): raise exceptions.ValidationError('Tags must set a list of strings') self.base.attributes.set('tags', tags) - def set_orbitals(self, **kwargs): + def set_orbitals(self, **kwargs): # type: ignore[override] """This method is inherited from OrbitalData, but is blocked here. If used will raise a NotImplementedError """ diff --git a/src/aiida/orm/nodes/data/list.py b/src/aiida/orm/nodes/data/list.py index 9fdd6ec866..d2c0857b35 100644 --- a/src/aiida/orm/nodes/data/list.py +++ b/src/aiida/orm/nodes/data/list.py @@ -9,6 +9,7 @@ """`Data` sub class to represent a list.""" from collections.abc import MutableSequence +from typing import Any from .base import to_aiida_type from .data import Data @@ -81,15 +82,15 @@ def remove(self, value): self.set_list(data) return item - def pop(self, **kwargs): + def pop(self, index: int = -1) -> Any: """Remove and return item at index (default last).""" data = self.get_list() - item = data.pop(**kwargs) + item = data.pop(index) if not self._using_list_reference(): self.set_list(data) return item - def index(self, value): + def index(self, value: Any, start: int = 0, stop: int = 0) -> int: """Return first index of value..""" return self.get_list().index(value) diff --git a/src/aiida/orm/nodes/data/singlefile.py b/src/aiida/orm/nodes/data/singlefile.py index c0f3797f24..8faefc8cb4 100644 --- a/src/aiida/orm/nodes/data/singlefile.py +++ b/src/aiida/orm/nodes/data/singlefile.py @@ -71,9 +71,7 @@ def open(self, path: FilePath, mode: t.Literal['rb']) -> t.Iterator[t.BinaryIO]: @t.overload @contextlib.contextmanager - def open( # type: ignore[overload-overlap] - self, path: None = None, mode: t.Literal['r'] = ... - ) -> t.Iterator[t.TextIO]: ... + def open(self, path: None = None, mode: t.Literal['r'] = ...) -> t.Iterator[t.TextIO]: ... @t.overload @contextlib.contextmanager diff --git a/src/aiida/plugins/utils.py b/src/aiida/plugins/utils.py index c284b25912..7d1e16a363 100644 --- a/src/aiida/plugins/utils.py +++ b/src/aiida/plugins/utils.py @@ -38,7 +38,7 @@ def __init__(self): def logger(self) -> Logger: return self._logger - def get_version_info(self, plugin: str | type) -> dict[t.Any, dict[t.Any, t.Any]]: + def get_version_info(self, plugin: str | t.Any) -> dict[t.Any, dict[t.Any, t.Any]]: """Get the version information for a given plugin. .. note:: diff --git a/src/aiida/storage/sqlite_dos/backend.py b/src/aiida/storage/sqlite_dos/backend.py index 7be70f4a1c..2443fee259 100644 --- a/src/aiida/storage/sqlite_dos/backend.py +++ b/src/aiida/storage/sqlite_dos/backend.py @@ -26,7 +26,7 @@ from aiida.common import exceptions from aiida.common.log import AIIDA_LOGGER from aiida.manage.configuration.profile import Profile -from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER +from aiida.manage.configuration.settings import AiiDAConfigDir from aiida.orm.implementation import BackendEntity from aiida.storage.log import MIGRATE_LOGGER from aiida.storage.psql_dos.models.settings import DbSetting @@ -203,7 +203,7 @@ class Model(BaseModel, defer_build=True): filepath: str = Field( title='Directory of the backend', description='Filepath of the directory in which to store data for this backend.', - default_factory=lambda: str(AIIDA_CONFIG_FOLDER / 'repository' / f'sqlite_dos_{uuid4().hex}'), + default_factory=lambda: str(AiiDAConfigDir.get() / 'repository' / f'sqlite_dos_{uuid4().hex}'), ) @field_validator('filepath') diff --git a/src/aiida/tools/pytest_fixtures/configuration.py b/src/aiida/tools/pytest_fixtures/configuration.py index ca38db3fb0..ed06072b71 100644 --- a/src/aiida/tools/pytest_fixtures/configuration.py +++ b/src/aiida/tools/pytest_fixtures/configuration.py @@ -10,6 +10,8 @@ import pytest +from aiida.manage.configuration.settings import AiiDAConfigDir + if t.TYPE_CHECKING: from aiida.manage.configuration.config import Config @@ -53,7 +55,7 @@ def factory(dirpath: pathlib.Path): dirpath_config = dirpath / settings.DEFAULT_CONFIG_DIR_NAME os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = str(dirpath_config) - settings.set_configuration_directory(dirpath_config) + AiiDAConfigDir.set(dirpath_config) config = get_config(create=True) try: @@ -61,7 +63,7 @@ def factory(dirpath: pathlib.Path): finally: if current_config: reset_config() - settings.set_configuration_directory(pathlib.Path(current_config.dirpath)) + AiiDAConfigDir.set(pathlib.Path(current_config.dirpath)) get_config() if current_path_variable is None: diff --git a/src/aiida/tools/pytest_fixtures/daemon.py b/src/aiida/tools/pytest_fixtures/daemon.py index 74e3620193..89ef02d841 100644 --- a/src/aiida/tools/pytest_fixtures/daemon.py +++ b/src/aiida/tools/pytest_fixtures/daemon.py @@ -116,7 +116,7 @@ def test(submit_and_await): from aiida.engine import ProcessState def factory( - submittable: 'Process' | 'ProcessBuilder' | 'ProcessNode', + submittable: type[Process] | ProcessBuilder | ProcessNode, state: ProcessState = ProcessState.FINISHED, timeout: int = 20, **kwargs, diff --git a/tests/cmdline/groups/test_dynamic.py b/tests/cmdline/groups/test_dynamic.py index 3c741b8fe2..48c18f0941 100644 --- a/tests/cmdline/groups/test_dynamic.py +++ b/tests/cmdline/groups/test_dynamic.py @@ -17,7 +17,7 @@ class Model(BaseModel): union_type: t.Union[int, float] = Field(title='Union type') without_default: str = Field(title='Without default') with_default: str = Field(title='With default', default='default') - with_default_factory: str = Field(title='With default factory', default_factory=lambda: True) + with_default_factory: str = Field(title='With default factory', default_factory=lambda: True) # type: ignore[assignment] def test_list_options(entry_points): diff --git a/tests/cmdline/utils/test_ascii_vis.py b/tests/cmdline/utils/test_ascii_vis.py new file mode 100644 index 0000000000..9fb6d26423 --- /dev/null +++ b/tests/cmdline/utils/test_ascii_vis.py @@ -0,0 +1,20 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Tests for the :mod:`aiida.cmdline.utils.ascii_vis` module.""" + +from aiida.orm.nodes.process.process import ProcessNode + + +def test_build_call_graph(): + from aiida.cmdline.utils.ascii_vis import build_call_graph + + node = ProcessNode() + + call_graph = build_call_graph(node) + assert call_graph == 'None None' diff --git a/tests/conftest.py b/tests/conftest.py index 2166cc06c0..b50cf90843 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,7 @@ import types import typing as t import warnings +from enum import Enum from pathlib import Path import click @@ -37,7 +38,14 @@ pytest_plugins = ['aiida.tools.pytest_fixtures', 'sphinx.testing.fixtures'] -def pytest_collection_modifyitems(items): +class TestDbBackend(Enum): + """Options for the '--db-backend' CLI argument when running pytest.""" + + SQLITE = 'sqlite' + PSQL = 'psql' + + +def pytest_collection_modifyitems(items, config): """Automatically generate markers for certain tests. Most notably, we add the 'presto' marker for all tests that @@ -47,6 +55,14 @@ def pytest_collection_modifyitems(items): filepath_django = Path(__file__).parent / 'storage' / 'psql_dos' / 'migrations' / 'django_branch' filepath_sqla = Path(__file__).parent / 'storage' / 'psql_dos' / 'migrations' / 'sqlalchemy_branch' + # If the user requested the SQLite backend, automatically skip incompatible tests + if config.option.db_backend is TestDbBackend.SQLITE: + if config.option.markexpr != '': + # Don't overwrite markers that the user already provided via '-m ' cmdline argument + config.option.markexpr += ' and (not requires_psql)' + else: + config.option.markexpr = 'not requires_psql' + for item in items: filepath_item = Path(item.fspath) @@ -68,6 +84,30 @@ def pytest_collection_modifyitems(items): item.add_marker('presto') +def db_backend_type(string): + """Conversion function for the custom '--db-backend' pytest CLI option + + :param string: String provided by the user via CLI + :returns: DbBackend enum corresponding to user input string + """ + try: + return TestDbBackend(string) + except ValueError: + msg = f"Invalid --db-backend option '{string}'\nMust be one of: {tuple(db.value for db in TestDbBackend)}" + raise pytest.UsageError(msg) + + +def pytest_addoption(parser): + parser.addoption( + '--db-backend', + action='store', + default=TestDbBackend.SQLITE, + required=False, + help=f'Database backend to be used for tests {tuple(db.value for db in TestDbBackend)}', + type=db_backend_type, + ) + + @pytest.fixture(scope='session') def aiida_profile(pytestconfig, aiida_config, aiida_profile_factory, config_psql_dos, config_sqlite_dos): """Create and load a profile with ``core.psql_dos`` as a storage backend and RabbitMQ as the broker. @@ -77,18 +117,22 @@ def aiida_profile(pytestconfig, aiida_config, aiida_profile_factory, config_psql be run against the main storage backend, which is ``core.sqlite_dos``. """ marker_opts = pytestconfig.getoption('-m') + db_backend = pytestconfig.getoption('--db-backend') - # By default we use RabbitMQ broker and psql_dos storage + # We use RabbitMQ broker by default unless 'presto' marker is specified broker = 'core.rabbitmq' if 'not requires_rmq' in marker_opts or 'presto' in marker_opts: broker = None - if 'not requires_psql' in marker_opts or 'presto' in marker_opts: + if db_backend is TestDbBackend.SQLITE: storage = 'core.sqlite_dos' config = config_sqlite_dos() - else: + elif db_backend is TestDbBackend.PSQL: storage = 'core.psql_dos' config = config_psql_dos() + else: + # This should be unreachable + raise ValueError(f'Invalid DB backend {db_backend}') with aiida_profile_factory( aiida_config, storage_backend=storage, storage_config=config, broker_backend=broker @@ -329,7 +373,7 @@ def empty_config(tmp_path) -> Config: """ from aiida.common.utils import Capturing from aiida.manage import configuration, get_manager - from aiida.manage.configuration import settings + from aiida.manage.configuration.settings import AiiDAConfigDir manager = get_manager() @@ -343,7 +387,7 @@ def empty_config(tmp_path) -> Config: # Set the configuration directory to a temporary directory. This will create the necessary folders for an empty # AiiDA configuration and set relevant global variables in :mod:`aiida.manage.configuration.settings`. - settings.set_configuration_directory(tmp_path) + AiiDAConfigDir.set(tmp_path) # The constructor of `Config` called by `load_config` will print warning messages about migrating it with Capturing(): @@ -361,7 +405,7 @@ def empty_config(tmp_path) -> Config: # like the :class:`aiida.engine.daemon.client.DaemonClient` will not function properly after a test that uses # this fixture because the paths of the daemon files would still point to the path of the temporary config # folder created by this fixture. - settings.set_configuration_directory(pathlib.Path(current_config_path)) + AiiDAConfigDir.set(pathlib.Path(current_config_path)) # Reload the original profile manager.load_profile(current_profile_name) diff --git a/tests/manage/configuration/test_config.py b/tests/manage/configuration/test_config.py index b1decb09a7..7fedb4bf2a 100644 --- a/tests/manage/configuration/test_config.py +++ b/tests/manage/configuration/test_config.py @@ -19,6 +19,7 @@ from aiida.manage.configuration.config import Config from aiida.manage.configuration.migrations import CURRENT_CONFIG_VERSION, OLDEST_COMPATIBLE_CONFIG_VERSION from aiida.manage.configuration.options import get_option +from aiida.manage.configuration.settings import AiiDAConfigDir from aiida.storage.sqlite_temp import SqliteTempBackend @@ -42,7 +43,7 @@ def cache_aiida_path_variable(): # Make sure to reset the global variables set by the following call that are dependent on the environment variable # ``DEFAULT_AIIDA_PATH_VARIABLE``. It may have been changed by a test using this fixture. - settings.set_configuration_directory() + AiiDAConfigDir.set() @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder') @@ -65,11 +66,11 @@ def test_environment_variable_not_set(chdir_tmp_path, monkeypatch): del os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] except KeyError: pass - settings.set_configuration_directory() + AiiDAConfigDir.set() config_folder = chdir_tmp_path / settings.DEFAULT_CONFIG_DIR_NAME assert os.path.isdir(config_folder) - assert settings.AIIDA_CONFIG_FOLDER == pathlib.Path(config_folder) + assert AiiDAConfigDir.get() == pathlib.Path(config_folder) @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder') @@ -78,12 +79,12 @@ def test_environment_variable_set_single_path_without_config_folder(tmp_path): """If `AIIDA_PATH` is set but does not contain a configuration folder, it should be created.""" # Set the environment variable and call configuration initialization os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = str(tmp_path) - settings.set_configuration_directory() + AiiDAConfigDir.set() # This should have created the configuration directory in the path config_folder = tmp_path / settings.DEFAULT_CONFIG_DIR_NAME assert config_folder.is_dir() - assert settings.AIIDA_CONFIG_FOLDER == config_folder + assert AiiDAConfigDir.get() == config_folder @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder') @@ -94,12 +95,12 @@ def test_environment_variable_set_single_path_with_config_folder(tmp_path): # Set the environment variable and call configuration initialization os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = str(tmp_path) - settings.set_configuration_directory() + AiiDAConfigDir.set() # This should have created the configuration directory in the path config_folder = tmp_path / settings.DEFAULT_CONFIG_DIR_NAME assert config_folder.is_dir() - assert settings.AIIDA_CONFIG_FOLDER == config_folder + assert AiiDAConfigDir.get() == config_folder @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder') @@ -114,12 +115,12 @@ def test_environment_variable_path_including_config_folder(tmp_path): """ # Set the environment variable with a path that include base folder name and call config initialization os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = str(tmp_path / settings.DEFAULT_CONFIG_DIR_NAME) - settings.set_configuration_directory() + AiiDAConfigDir.set() # This should have created the configuration directory in the pathpath config_folder = tmp_path / settings.DEFAULT_CONFIG_DIR_NAME assert config_folder.is_dir() - assert settings.AIIDA_CONFIG_FOLDER == config_folder + assert AiiDAConfigDir.get() == config_folder @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder') @@ -137,12 +138,12 @@ def test_environment_variable_set_multiple_path(tmp_path): # Set the environment variable to contain three paths and call configuration initialization env_variable = f'{directory_a}:{directory_b}:{directory_c}' os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = env_variable - settings.set_configuration_directory() + AiiDAConfigDir.set() # This should have created the configuration directory in the last path config_folder = directory_c / settings.DEFAULT_CONFIG_DIR_NAME assert os.path.isdir(config_folder) - assert settings.AIIDA_CONFIG_FOLDER == config_folder + assert AiiDAConfigDir.get() == config_folder def compare_config_in_memory_and_on_disk(config, filepath): @@ -152,9 +153,7 @@ def compare_config_in_memory_and_on_disk(config, filepath): :param filepath: absolute filepath to a configuration file :raises AssertionError: if content of `config` is not equal to that of file on disk """ - from aiida.manage.configuration.settings import DEFAULT_CONFIG_INDENT_SIZE - - in_memory = json.dumps(config.dictionary, indent=DEFAULT_CONFIG_INDENT_SIZE) + in_memory = json.dumps(config.dictionary, indent=settings.DEFAULT_CONFIG_INDENT_SIZE) # Read the content stored on disk with open(filepath, 'r', encoding='utf8') as handle: diff --git a/tests/manage/test_caching_config.py b/tests/manage/test_caching_config.py index 50a2e4ac48..932a32873c 100644 --- a/tests/manage/test_caching_config.py +++ b/tests/manage/test_caching_config.py @@ -45,11 +45,12 @@ def test_merge_deprecated_yaml(tmp_path): """ from aiida.common.warnings import AiidaDeprecationWarning from aiida.manage import configuration, get_manager - from aiida.manage.configuration import get_config_option, load_profile, settings + from aiida.manage.configuration import get_config_option, load_profile + from aiida.manage.configuration.settings import AiiDAConfigDir # Store the current configuration instance and config directory path current_config = configuration.CONFIG - current_config_path = current_config.dirpath + current_config_path = pathlib.Path(current_config.dirpath) current_profile_name = configuration.get_profile().name try: @@ -57,7 +58,7 @@ def test_merge_deprecated_yaml(tmp_path): configuration.CONFIG = None # Create a temporary folder, set it as the current config directory path - settings.AIIDA_CONFIG_FOLDER = str(tmp_path) + AiiDAConfigDir.set(pathlib.Path(tmp_path)) config_dictionary = json.loads( pathlib.Path(__file__) .parent.joinpath('configuration/migrations/test_samples/reference/6.json') @@ -86,7 +87,7 @@ def test_merge_deprecated_yaml(tmp_path): # Reset the config folder path and the config instance. Note this will always be executed after the yield no # matter what happened in the test that used this fixture. get_manager().unload_profile() - settings.AIIDA_CONFIG_FOLDER = current_config_path + AiiDAConfigDir.set(current_config_path) configuration.CONFIG = current_config load_profile(current_profile_name) diff --git a/tests/orm/test_querybuilder.py b/tests/orm/test_querybuilder.py index e39f20a7b9..8797fe4e03 100644 --- a/tests/orm/test_querybuilder.py +++ b/tests/orm/test_querybuilder.py @@ -9,6 +9,7 @@ """Tests for the QueryBuilder.""" import copy +import json import uuid import warnings from collections import defaultdict @@ -1702,3 +1703,163 @@ def test_statistics_default_class(self, aiida_localhost): # data are correct res = next(iter(qb.dict()[0].values())) assert res == expected_dict + + +class TestJsonFilters: + @pytest.mark.parametrize( + 'data,filters,is_match', + ( + # contains different types of element + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': [1]}}, True), + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': ['2']}}, True), + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': [None]}}, True), + # contains multiple elements of various types + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': [1, None]}}, True), + # contains non-exist elements + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': [114514]}}, False), + # contains empty set + ({'arr': [1, '2', None]}, {'attributes.arr': {'contains': []}}, True), + ({'arr': []}, {'attributes.arr': {'contains': []}}, True), + # nested arrays + ({'arr': [[1, 0], [0, 2]]}, {'attributes.arr': {'contains': [[1, 0]]}}, True), + ({'arr': [[2, 3], [0, 1], []]}, {'attributes.arr': {'contains': [[1, 0]]}}, True), + ({'arr': [[2, 3], [1]]}, {'attributes.arr': {'contains': [[4]]}}, False), + ({'arr': [[1, 0], [0, 2]]}, {'attributes.arr': {'contains': [[3]]}}, False), + ({'arr': [[1, 0], [0, 2]]}, {'attributes.arr': {'contains': [3]}}, False), + ({'arr': [[1, 0], [0, 2]]}, {'attributes.arr': {'contains': [[2]]}}, True), + ({'arr': [[1, 0], [0, 2]]}, {'attributes.arr': {'contains': [2]}}, False), + ({'arr': [[1, 0], [0, 2], 3]}, {'attributes.arr': {'contains': [[3]]}}, False), + ({'arr': [[1, 0], [0, 2], 3]}, {'attributes.arr': {'contains': [3]}}, True), + # negations + ({'arr': [1, '2', None]}, {'attributes.arr': {'!contains': [1]}}, False), + ({'arr': [1, '2', None]}, {'attributes.arr': {'!contains': []}}, False), + ({'arr': [1, '2', None]}, {'attributes.arr': {'!contains': [114514]}}, True), + ({'arr': [1, '2', None]}, {'attributes.arr': {'!contains': [1, 114514]}}, True), + # TODO: these pass, but why? are these behaviors expected? + # non-exist `attr_key`s + ({'foo': []}, {'attributes.arr': {'contains': []}}, False), + ({'foo': []}, {'attributes.arr': {'!contains': []}}, False), + ), + ids=json.dumps, + ) + @pytest.mark.usefixtures('aiida_profile_clean') + @pytest.mark.requires_psql + def test_json_filters_contains_arrays(self, data, filters, is_match): + """Test QueryBuilder filter `contains` for JSON array fields""" + orm.Dict(data).store() + qb = orm.QueryBuilder().append(orm.Dict, filters=filters) + assert qb.count() in {0, 1} + found = qb.count() == 1 + assert found == is_match + + @pytest.mark.parametrize( + 'data,filters,is_match', + ( + # contains different types of values + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'contains': {'k1': 1}}}, + True, + ), + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'contains': {'k1': 1, 'k2': '2'}}}, + True, + ), + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'contains': {'k3': None}}}, + True, + ), + # contains empty set + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'contains': {}}}, + True, + ), + # doesn't contain non-exist entries + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'contains': {'k1': 1, 'k': 'v'}}}, + False, + ), + # negations + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'!contains': {'k1': 1}}}, + False, + ), + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'!contains': {'k1': 1, 'k': 'v'}}}, + True, + ), + ( + { + 'dict': { + 'k1': 1, + 'k2': '2', + 'k3': None, + } + }, + {'attributes.dict': {'!contains': {}}}, + False, + ), + # TODO: these pass, but why? are these behaviors expected? + # non-exist `attr_key`s + ({'map': {}}, {'attributes.dict': {'contains': {}}}, False), + ({'map': {}}, {'attributes.dict': {'!contains': {}}}, False), + ), + ids=json.dumps, + ) + @pytest.mark.usefixtures('aiida_profile_clean') + @pytest.mark.requires_psql + def test_json_filters_contains_object(self, data, filters, is_match): + """Test QueryBuilder filter `contains` for JSON object fields""" + orm.Dict(data).store() + qb = orm.QueryBuilder().append(orm.Dict, filters=filters) + assert qb.count() in {0, 1} + found = qb.count() == 1 + assert found == is_match diff --git a/tests/storage/psql_dos/conftest.py b/tests/storage/psql_dos/conftest.py index 16136b8df9..c24db0ac72 100644 --- a/tests/storage/psql_dos/conftest.py +++ b/tests/storage/psql_dos/conftest.py @@ -13,17 +13,12 @@ from aiida.common.exceptions import MissingConfigurationError from aiida.manage.configuration import get_config +STORAGE_BACKEND_ENTRY_POINT = None try: if test_profile := os.environ.get('AIIDA_TEST_PROFILE'): STORAGE_BACKEND_ENTRY_POINT = get_config().get_profile(test_profile).storage_backend - # TODO: The else branch is wrong - else: - STORAGE_BACKEND_ENTRY_POINT = 'core.psql_dos' -except MissingConfigurationError: - # TODO: This is actually not true anymore! - # Case when ``pytest`` is invoked without existing config, in which case it will rely on the automatic test profile - # creation which currently always uses ``core.psql_dos`` for the storage backend - STORAGE_BACKEND_ENTRY_POINT = 'core.psql_dos' +except MissingConfigurationError as e: + raise ValueError(f"Could not parse configuration of AiiDA test profile '{test_profile}'") from e -if STORAGE_BACKEND_ENTRY_POINT != 'core.psql_dos': +if STORAGE_BACKEND_ENTRY_POINT is not None and STORAGE_BACKEND_ENTRY_POINT != 'core.psql_dos': collect_ignore_glob = ['*'] diff --git a/tests/storage/psql_dos/migrations/django_branch/test_migrate_to_head.py b/tests/storage/psql_dos/migrations/django_branch/test_migrate_to_head.py index 69c7e643d9..11f9fb8c3a 100644 --- a/tests/storage/psql_dos/migrations/django_branch/test_migrate_to_head.py +++ b/tests/storage/psql_dos/migrations/django_branch/test_migrate_to_head.py @@ -11,6 +11,15 @@ from aiida.storage.psql_dos.migrator import PsqlDosMigrator +def test_all_tests_marked_as_nightly(request): + """Test that all tests in this folder are tagged with 'nightly' marker""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 2 + assert 'nightly' in own_markers + assert 'requires_psql' in own_markers + + def test_migrate(perform_migrations: PsqlDosMigrator): """Test that the migrator can migrate from the base of the django branch, to the main head.""" perform_migrations.migrate_up('django@django_0001') # the base of the django branch diff --git a/tests/storage/psql_dos/test_backend.py b/tests/storage/psql_dos/test_backend.py index 6fe35ac747..bbc77b1a14 100644 --- a/tests/storage/psql_dos/test_backend.py +++ b/tests/storage/psql_dos/test_backend.py @@ -13,6 +13,14 @@ from aiida.orm import User +def test_all_tests_marked_with_requires_psql(request): + """Test that all tests in this folder are marked with 'requires_psql'""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 1 + assert own_markers[0] == 'requires_psql' + + @pytest.mark.usefixtures('aiida_profile_clean') def test_default_user(): assert isinstance(get_manager().get_profile_storage().default_user, User) diff --git a/tests/test_markers.py b/tests/test_markers.py new file mode 100644 index 0000000000..d9d6e63871 --- /dev/null +++ b/tests/test_markers.py @@ -0,0 +1,64 @@ +"""Tests markers that have custom in conftest.py""" + +import pytest + + +def test_presto_auto_mark(request): + """Test that the presto marker is added automatically""" + own_markers = [marker.name for marker in request.node.own_markers] + assert len(own_markers) == 1 + assert own_markers[0] == 'presto' + + +@pytest.mark.sphinx +def test_presto_mark_and_another_mark(request): + """Test that presto marker is added even if there is an existing marker (except requires_rmq|psql)""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 2 + assert 'presto' in own_markers + assert 'sphinx' in own_markers + + +@pytest.mark.requires_rmq +def test_no_presto_mark_if_rmq(request): + """Test that presto marker is NOT added if the test is mark with "requires_rmq""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 1 + assert own_markers[0] == 'requires_rmq' + + +@pytest.mark.requires_psql +def test_no_presto_mark_if_psql(request): + """Test that presto marker is NOT added if the test is mark with "requires_psql""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 1 + assert own_markers[0] == 'requires_psql' + + +@pytest.mark.nightly +def test_no_presto_mark_if_nightly(request): + """Test that presto marker is NOT added if the test is mark with "requires_psql""" + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 1 + assert own_markers[0] == 'nightly' + + +@pytest.mark.requires_psql +def test_requires_psql_with_sqlite_impossible(pytestconfig): + db_backend = pytestconfig.getoption('--db-backend') + if db_backend.value == 'sqlite': + pytest.fail('This test should not have been executed with SQLite backend!') + + +def test_daemon_client_fixture_automarked(request, daemon_client): + """Test that any test using ``daemon_client`` fixture is + automatically tagged with 'requires_rmq' mark + """ + own_markers = [marker.name for marker in request.node.own_markers] + + assert len(own_markers) == 1 + assert own_markers[0] == 'requires_rmq' diff --git a/utils/patch-release.sh b/utils/patch-release.sh new file mode 100755 index 0000000000..a9d67bee79 --- /dev/null +++ b/utils/patch-release.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Script: patch-release.sh +# Description: +# Cherry-picks a list of commits, amends each with the original commit hash for tracking, +# and generates a summary from the short github commit messages with links to each commit. +# +# Usage: +# ./patch-release.sh ... +# +# Example: +# ./patch-release.sh abc1234 def5678 + +set -e + +# Check if at least two arguments are provided (repo and at least one commit) +if [ "$#" -lt 1 ]; then + echo "Usage: $0 ..." + echo "Example: $0 abc1234 def5678" + exit 1 +fi + +GITHUB_REPO="aiidateam/aiida-core" + +# Create an array to store commit summaries +declare -a commit_summaries=() + +# Loop through each commit hash +for commit in "$@"; do + # Cherry-pick the commit + if git cherry-pick "$commit"; then + # If cherry-pick succeeds, get the short message and short hash + commit_message=$(git log -1 --pretty=format:"%B" HEAD) + original_short_hash=$(git log -1 --pretty=format:"%h" "$commit") + original_long_hash=$(git rev-parse $original_short_hash) + + # Amend the cherry-picked commit to include the original commit ID for tracking + git commit --amend -m "$commit_message" -m "Cherry-pick: $original_long_hash" + + # Format the output as a Markdown list item and add to the array + short_commit_message=$(git log -1 --pretty=format:"%s" HEAD) + cherry_picked_hash=$(git log -1 --pretty=format:"%h" HEAD) + commit_summaries+=("- $short_commit_message [[${commit}]](https://github.com/$GITHUB_REPO/commit/${original_long_hash})") + else + echo "Failed to cherry-pick commit $commit" + # Abort the cherry-pick in case of conflict + git cherry-pick --abort + exit 1 + fi +done + +# Print the summary +echo -e "\n### Cherry-Picked Commits Summary:\n" +for summary in "${commit_summaries[@]}"; do + echo "$summary" +done