Merge branch 'main' into xdist

.github/workflows/benchmark.yml

@@ -56,7 +56,7 @@ jobs:
         python-version: '3.10'
 
     - name: Run benchmarks
-      run: pytest --benchmark-only --benchmark-json benchmark.json
+      run: pytest --db-backend psql --benchmark-only --benchmark-json benchmark.json tests/
 
     - name: Store benchmark result
       uses: aiidateam/github-action-benchmark@v3
@@ -73,4 +73,4 @@ jobs:
         alert-threshold: 200%
         comment-on-alert: true
         fail-on-alert: false
-        alert-comment-cc-users: '@chrisjsewell,@giovannipizzi'
+        alert-comment-cc-users: '@giovannipizzi,@agoscinski,@GeigerJ2,@khsrali,@unkcpz'

.github/workflows/ci-code.yml

@@ -104,7 +104,7 @@ jobs:
         AIIDA_WARN_v3: 1
       # Python 3.12 has a performance regression when running with code coverage
       # so run code coverage only for python 3.9.
-      run: pytest -v tests -n auto -m 'not nightly' ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
+      run: pytest -n auto --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
 
     - name: Upload coverage report
       if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core'
@@ -139,7 +139,7 @@ jobs:
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -n auto -m 'presto'
+      run: pytest -n auto -m 'presto' tests/
 
 
   verdi:

.github/workflows/nightly.yml

@@ -104,19 +104,6 @@ jobs:
         rabbitmq-version: ['3.11', '3.12', '3.13']
 
     services:
-      postgres:
-        image: postgres:16
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:${{ matrix.rabbitmq-version }}-management
         ports:
@@ -132,9 +119,6 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install system dependencies
-      run: sudo apt update && sudo apt install postgresql
-
     - name: Setup SSH on localhost
       run: .github/workflows/setup_ssh.sh
 
@@ -145,7 +129,7 @@ jobs:
       id: tests
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -sv -m 'requires_rmq'
+      run: pytest -s --db-backend sqlite -m 'requires_rmq' tests/
 
     - name: Slack notification
       # Always run this step (otherwise it would be skipped if any of the previous steps fail) but only if the

.github/workflows/release.yml

@@ -54,19 +54,6 @@ jobs:
     timeout-minutes: 30
 
     services:
-      postgres:
-        image: postgres:10
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:3.8.14-management
         ports:
@@ -76,16 +63,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Install system dependencies
-      run: |
-        sudo apt update
-        sudo apt install postgresql graphviz
-
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
 
     - name: Run sub-set of test suite
-      run: pytest -sv -k 'requires_rmq'
+      run: pytest -s -m requires_rmq --db-backend=sqlite tests/
 
   publish:
 

.github/workflows/test-install.yml

@@ -229,7 +229,7 @@ jobs:
       env:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
-      run: pytest --verbose tests -m 'not nightly'
+      run: pytest --db-backend psql tests -m 'not nightly' tests/
 
     - name: Freeze test environment
       run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt

.github/workflows/tests_nightly.sh

@@ -13,4 +13,4 @@ verdi -p test_aiida run ${SYSTEM_TESTS}/test_containerized_code.py
 bash ${SYSTEM_TESTS}/test_polish_workchains.sh
 verdi daemon stop
 
-AIIDA_TEST_PROFILE=test_aiida pytest -v tests -m 'nightly'
+AIIDA_TEST_PROFILE=test_aiida pytest --db-backend psql -m nightly tests/

.pre-commit-config.yaml

@@ -90,6 +90,7 @@ repos:
         .docker/.*|
         docs/.*|
         utils/.*|
+        tests/.*|
 
         src/aiida/calculations/arithmetic/add.py|
         src/aiida/calculations/diff_tutorial/calculations.py|
@@ -124,7 +125,6 @@ repos:
         src/aiida/engine/processes/ports.py|
         src/aiida/manage/configuration/__init__.py|
         src/aiida/manage/configuration/config.py|
-        src/aiida/manage/configuration/profile.py|
         src/aiida/manage/external/rmq/launcher.py|
         src/aiida/manage/tests/main.py|
         src/aiida/manage/tests/pytest_fixtures.py|
@@ -192,17 +192,6 @@ repos:
         src/aiida/transports/plugins/local.py|
         src/aiida/transports/plugins/ssh.py|
         src/aiida/workflows/arithmetic/multiply_add.py|
-
-        tests/conftest.py|
-        tests/repository/conftest.py|
-        tests/repository/test_repository.py|
-        tests/sphinxext/sources/workchain/conf.py|
-        tests/sphinxext/sources/workchain_broken/conf.py|
-        tests/storage/psql_dos/migrations/conftest.py|
-        tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py|
-        tests/test_calculation_node.py|
-        tests/test_nodes.py|
-
       )$
 
   - id: dm-generate-all

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v2.6.3 - 2024-11-6
+
+### Fixes
+- CLI: Fix exception for `verdi plugin list` (#6560) [[c3b10b7]](https://github.com/aiidateam/aiida-core/commit/c3b10b759a9cd062800ef120591d5c7fd0ae4ee7)
+- `DirectScheduler`: Ensure killing child processes (#6572) [[fddffca]](https://github.com/aiidateam/aiida-core/commit/fddffca67b4f7e3b76b19df7db8e1511c449d2d9)   
+- Engine: Fix state change broadcast before process node is updated  (#6580) [[867353c]](https://github.com/aiidateam/aiida-core/commit/867353c415c61d94a2427d5225dd5224a1b95fb9)
+
+### Devops
+- Docker: Replace sleep with `s6-notifyoncheck` (#6475) [[9579378b]](https://github.com/aiidateam/aiida-core/commit/9579378ba063237baa5b73380eb8e9f0a28529ee)
+- Fix failed docker CI using more reasoning grep regex to parse python version (#6581) [[332a4a91]](https://github.com/aiidateam/aiida-core/commit/332a4a915771afedcb144463b012558e4669e529)
+- DevOps: Fix json query in reading the docker names to filter out fields not starting with aiida (#6573) [[e1467edc]](https://github.com/aiidateam/aiida-core/commit/e1467edca902867e53605e0e60b67f8767bf8d3e)
+
+
 ## v2.6.2 - 2024-08-07
 
 ### Fixes
@@ -31,7 +44,7 @@
 
 ## v2.6.1 - 2024-07-01
 
-### Fixes:
+### Fixes
 - Fixtures: Make `pgtest` truly an optional dependency [[9fe8fd2e0]](https://github.com/aiidateam/aiida-core/commit/9fe8fd2e0b88e746ee2156eccb71b7adbab6b2c5)
 
 

pyproject.toml

@@ -211,7 +211,7 @@ notebook = [
 ]
 pre-commit = [
   'aiida-core[atomic_tools,rest,tests,tui]',
-  'mypy~=1.10.0',
+  'mypy~=1.13.0',
   'packaging~=23.0',
   'pre-commit~=3.5',
   'sqlalchemy[mypy]~=2.0',
@@ -256,6 +256,7 @@ runaiida = 'aiida.cmdline.commands.cmd_run:run'
 verdi = 'aiida.cmdline.commands.cmd_verdi:verdi'
 
 [project.urls]
+Changelog = 'https://github.com/aiidateam/aiida-core/blob/main/CHANGELOG.md'
 Documentation = 'https://aiida.readthedocs.io'
 Home = 'http://www.aiida.net/'
 Source = 'https://github.com/aiidateam/aiida-core'

src/aiida/cmdline/commands/cmd_presto.py

@@ -67,7 +67,7 @@ def detect_postgres_config(
     """
     import secrets
 
-    from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
+    from aiida.manage.configuration.settings import AiiDAConfigDir
     from aiida.manage.external.postgres import Postgres
 
     dbinfo = {
@@ -92,13 +92,15 @@ def detect_postgres_config(
     except Exception as exception:
         raise ConnectionError(f'Unable to automatically create the PostgreSQL user and database: {exception}')
 
+    aiida_config_folder = AiiDAConfigDir.get()
+
     return {
         'database_hostname': postgres_hostname,
         'database_port': postgres_port,
         'database_name': database_name,
         'database_username': database_username,
         'database_password': database_password,
-        'repository_uri': f'file://{AIIDA_CONFIG_FOLDER / "repository" / profile_name}',
+        'repository_uri': f'file://{aiida_config_folder / "repository" / profile_name}',
     }
 
 

src/aiida/cmdline/commands/cmd_profile.py

@@ -169,9 +169,9 @@ def profile_list():
         # This can happen for a fresh install and the `verdi setup` has not yet been run. In this case it is still nice
         # to be able to see the configuration directory, for instance for those who have set `AIIDA_PATH`. This way
         # they can at least verify that it is correctly set.
-        from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
+        from aiida.manage.configuration.settings import AiiDAConfigDir
 
-        echo.echo_report(f'configuration folder: {AIIDA_CONFIG_FOLDER}')
+        echo.echo_report(f'configuration folder: {AiiDAConfigDir.get()}')
         echo.echo_critical(str(exception))
     else:
         echo.echo_report(f'configuration folder: {config.dirpath}')

src/aiida/cmdline/commands/cmd_status.py

@@ -61,13 +61,14 @@ def verdi_status(print_traceback, no_rmq):
     from aiida.common.docs import URL_NO_BROKER
     from aiida.common.exceptions import ConfigurationError
     from aiida.engine.daemon.client import DaemonException, DaemonNotRunningException
-    from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
+    from aiida.manage.configuration.settings import AiiDAConfigDir
     from aiida.manage.manager import get_manager
 
     exit_code = ExitCode.SUCCESS
+    configure_directory = AiiDAConfigDir.get()
 
     print_status(ServiceStatus.UP, 'version', f'AiiDA v{__version__}')
-    print_status(ServiceStatus.UP, 'config', AIIDA_CONFIG_FOLDER)
+    print_status(ServiceStatus.UP, 'config', configure_directory)
 
     manager = get_manager()
 

src/aiida/cmdline/params/options/commands/setup.py

@@ -66,11 +66,12 @@ def get_repository_uri_default(ctx):
     """
     import os
 
-    from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
+    from aiida.manage.configuration.settings import AiiDAConfigDir
 
     validate_profile_parameter(ctx)
+    configure_directory = AiiDAConfigDir.get()
 
-    return os.path.join(AIIDA_CONFIG_FOLDER, 'repository', ctx.params['profile'].name)
+    return os.path.join(configure_directory, 'repository', ctx.params['profile'].name)
 
 
 def get_quicksetup_repository_uri(ctx, param, value):

src/aiida/cmdline/utils/ascii_vis.py

@@ -29,7 +29,7 @@ def calc_info(node, call_link_label: bool = False) -> str:
         raise TypeError(f'Unknown type: {type(node)}')
 
     process_label = node.process_label
-    process_state = node.process_state.value.capitalize()
+    process_state = 'None' if node.process_state is None else node.process_state.value.capitalize()
     exit_status = node.exit_status
 
     if call_link_label and (caller := node.caller):

src/aiida/common/lang.py

@@ -96,4 +96,4 @@ def __init__(self, getter: Callable[[SelfType], ReturnType]) -> None:
         self.getter = getter
 
     def __get__(self, instance: Any, owner: SelfType) -> ReturnType:
-        return self.getter(owner)
+        return self.getter(owner)  # type: ignore[arg-type]

src/aiida/common/pydantic.py

@@ -41,7 +41,7 @@ class Model(BaseModel):
     field_info = Field(default, **kwargs)
 
     for key, value in (('priority', priority), ('short_name', short_name), ('option_cls', option_cls)):
-        if value is not None:
+        if value is not None and field_info is not None:
             field_info.metadata.append({key: value})
 
     return field_info

src/aiida/engine/daemon/client.py

@@ -94,10 +94,10 @@ def __init__(self, profile: Profile):
         from aiida.common.docs import URL_NO_BROKER
 
         type_check(profile, Profile)
-        config = get_config()
+        self._config = get_config()
         self._profile = profile
         self._socket_directory: str | None = None
-        self._daemon_timeout: int = config.get_option('daemon.timeout', scope=profile.name)
+        self._daemon_timeout: int = self._config.get_option('daemon.timeout', scope=profile.name)
 
         if self._profile.process_control_backend is None:
             raise ConfigurationError(
@@ -156,31 +156,31 @@ def virtualenv(self) -> str | None:
 
     @property
     def circus_log_file(self) -> str:
-        return self.profile.filepaths['circus']['log']
+        return self._config.filepaths(self.profile)['circus']['log']
 
     @property
     def circus_pid_file(self) -> str:
-        return self.profile.filepaths['circus']['pid']
+        return self._config.filepaths(self.profile)['circus']['pid']
 
     @property
     def circus_port_file(self) -> str:
-        return self.profile.filepaths['circus']['port']
+        return self._config.filepaths(self.profile)['circus']['port']
 
     @property
     def circus_socket_file(self) -> str:
-        return self.profile.filepaths['circus']['socket']['file']
+        return self._config.filepaths(self.profile)['circus']['socket']['file']
 
     @property
     def circus_socket_endpoints(self) -> dict[str, str]:
-        return self.profile.filepaths['circus']['socket']
+        return self._config.filepaths(self.profile)['circus']['socket']
 
     @property
     def daemon_log_file(self) -> str:
-        return self.profile.filepaths['daemon']['log']
+        return self._config.filepaths(self.profile)['daemon']['log']
 
     @property
     def daemon_pid_file(self) -> str:
-        return self.profile.filepaths['daemon']['pid']
+        return self._config.filepaths(self.profile)['daemon']['pid']
 
     def get_circus_port(self) -> int:
         """Retrieve the port for the circus controller, which should be written to the circus port file.

src/aiida/engine/processes/calcjobs/calcjob.py

@@ -1062,7 +1062,7 @@ def presubmit(self, folder: Folder) -> CalcInfo:
 
         def encoder(obj):
             if dataclasses.is_dataclass(obj):
-                return dataclasses.asdict(obj)
+                return dataclasses.asdict(obj)  # type: ignore[arg-type]
             raise TypeError(f' {obj!r} is not JSON serializable')
 
         subfolder = folder.get_subfolder('.aiida', create=True)

src/aiida/engine/processes/workchains/restart.py

@@ -29,7 +29,7 @@
 
 
 def validate_handler_overrides(
-    process_class: 'BaseRestartWorkChain', handler_overrides: Optional[orm.Dict], ctx: 'PortNamespace'
+    process_class: type['BaseRestartWorkChain'], handler_overrides: Optional[orm.Dict], ctx: 'PortNamespace'
 ) -> Optional[str]:
     """Validator for the ``handler_overrides`` input port of the ``BaseRestartWorkChain``.