Centralised the creation of the Autogroup

This also remove an overzelous isinstance check, and
moves additional checks in a cached function that is run only when
storing the very first node (that needs to be put in an autogroup),
making storing of nodes faster (even if times oscillates so it's hard
to estimate exactly by how much).

Also, added logic to allow for concurrent creation of multiple groups
(and test). This fixes #997

.ci/workchains.py

@@ -68,8 +68,8 @@ def a_magic_unicorn_appeared(self, node):
     @process_handler(priority=400, exit_codes=ArithmeticAddCalculation.exit_codes.ERROR_NEGATIVE_NUMBER)
     def error_negative_sum(self, node):
         """What even is a negative number, how can I have minus three melons?!."""
-        self.ctx.inputs.x = Int(abs(node.inputs.x.value))
-        self.ctx.inputs.y = Int(abs(node.inputs.y.value))
+        self.ctx.inputs.x = Int(abs(node.inputs.x.value))  # pylint: disable=invalid-name
+        self.ctx.inputs.y = Int(abs(node.inputs.y.value))  # pylint: disable=invalid-name
         return ProcessHandlerReport(True)
 
 

.gitignore

@@ -20,6 +20,7 @@
 .cache
 .pytest_cache
 .coverage
+coverage.xml
 
 # Files created by RPN tests
 .ci/polish/polish_workchains/polish*

aiida/backends/testbase.py

@@ -99,7 +99,11 @@ def tearDown(self):
 
     def reset_database(self):
         """Reset the database to the default state deleting any content currently stored"""
+        from aiida.orm import autogroup
+
         self.clean_db()
+        if autogroup.CURRENT_AUTOGROUP is not None:
+            autogroup.CURRENT_AUTOGROUP.clear_group_cache()
         self.insert_data()
 
     @classmethod
@@ -109,7 +113,10 @@ def insert_data(cls):
         inserts default data into the database (which is for the moment a
         default computer).
         """
+        from aiida.orm import User
+
         cls.create_user()
+        User.objects.reset()
         cls.create_computer()
 
     @classmethod
@@ -180,7 +187,11 @@ def user_email(cls):  # pylint: disable=no-self-argument
     def tearDownClass(cls, *args, **kwargs):  # pylint: disable=arguments-differ
         # Double check for double security to avoid to run the tearDown
         # if this is not a test profile
+        from aiida.orm import autogroup
+
         check_if_tests_can_run()
+        if autogroup.CURRENT_AUTOGROUP is not None:
+            autogroup.CURRENT_AUTOGROUP.clear_group_cache()
         cls.clean_db()
         cls.clean_repository()
         cls.__backend_instance.tearDownClass_method(*args, **kwargs)

aiida/cmdline/commands/cmd_run.py

@@ -20,7 +20,6 @@
 from aiida.cmdline.params.options.multivalue import MultipleValueOption
 from aiida.cmdline.utils import decorators, echo
 from aiida.common.warnings import AiidaDeprecationWarning
-from aiida.orm import autogroup
 
 
 @contextlib.contextmanager
@@ -43,6 +42,8 @@ def update_environment(argv):
 
 def validate_entrypoint_string_or_all(ctx, param, value, allow_all=True):  # pylint: disable=unused-argument,invalid-name
     """Validate that `value` is a valid entrypoint string or the string 'all'."""
+    from aiida.orm import autogroup
+
     try:
         autogroup.Autogroup.validate(value, allow_all=allow_all)
     except Exception as exc:
@@ -55,51 +56,61 @@ def validate_entrypoint_string_or_all(ctx, param, value, allow_all=True):  # pyl
 @click.argument('scriptname', type=click.STRING)
 @click.argument('varargs', nargs=-1, type=click.UNPROCESSED)
 @click.option('--group/--no-group', default=True, show_default=True, help='Enables the autogrouping')
-@click.option('-l', '--group-label', type=click.STRING, required=False, help='Specify the label of the auto group')
+@click.option(
+    '-l',
+    '--group-label-prefix',
+    type=click.STRING,
+    required=False,
+    help='Specify the prefix of the label of the auto group (numbers might be automatically '
+    'appended to generate unique names per run)'
+)
 @click.option(
     '-n',
     '--group-name',
     type=click.STRING,
     required=False,
-    help='Specify the name of the auto group [DEPRECATED, USE --group-label instead]'
+    help='Specify the name of the auto group [DEPRECATED, USE --group-label-prefix instead]'
 )
 @click.option(
     '-e',
     '--exclude',
     cls=MultipleValueOption,
-    default=[],
+    default=lambda: [],
     help='Exclude these classes from auto grouping (use full entrypoint strings)',
     callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
 )
 @click.option(
     '-i',
     '--include',
     cls=MultipleValueOption,
-    default=['all'],
+    default=lambda: ['all'],
     help='Include these classes from auto grouping  (use full entrypoint strings or "all")',
     callback=validate_entrypoint_string_or_all
 )
 @click.option(
     '-E',
     '--excludesubclasses',
     cls=MultipleValueOption,
-    default=[],
+    default=lambda: [],
     help='Exclude these classes and their sub classes from auto grouping (use full entrypoint strings)',
     callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
 )
 @click.option(
     '-I',
     '--includesubclasses',
     cls=MultipleValueOption,
-    default=[],
+    default=lambda: [],
     help='Include these classes and their sub classes from auto grouping (use full entrypoint strings)',
     callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
 )
 @decorators.with_dbenv()
-def run(scriptname, varargs, group, group_label, group_name, exclude, excludesubclasses, include, includesubclasses):
+def run(
+    scriptname, varargs, group, group_label_prefix, group_name, exclude, excludesubclasses, include, includesubclasses
+):
     # pylint: disable=too-many-arguments,exec-used
     """Execute scripts with preloaded AiiDA environment."""
     from aiida.cmdline.utils.shell import DEFAULT_MODULES_LIST
+    from aiida.orm import autogroup
 
     # Prepare the environment for the script to be run
     globals_dict = {
@@ -121,11 +132,10 @@ def run(scriptname, varargs, group, group_label, group_name, exclude, excludesub
         group_label = group_name
 
     if group:
-        automatic_group_label = group_label
-
         aiida_verdilib_autogroup = autogroup.Autogroup()
-        if automatic_group_label is not None:
-            aiida_verdilib_autogroup.set_group_label(automatic_group_label)
+        # if group_label_prefix is None, use autogenerated name
+        if group_label_prefix is not None:
+            aiida_verdilib_autogroup.set_group_label_prefix(group_label_prefix)
         aiida_verdilib_autogroup.set_exclude(exclude)
         aiida_verdilib_autogroup.set_include(include)
         aiida_verdilib_autogroup.set_exclude_with_subclasses(excludesubclasses)

aiida/engine/processes/calcjobs/calcjob.py

@@ -64,7 +64,7 @@ def validate_calc_job(inputs, ctx):
         )
 
 
-def validate_parser(parser_name, ctx):
+def validate_parser(parser_name, ctx):  # pylint: disable=unused-argument
     """Validate the parser.
 
     :raises InputValidationError: if the parser name does not correspond to a loadable `Parser` class.
@@ -78,7 +78,7 @@ def validate_parser(parser_name, ctx):
             raise exceptions.InputValidationError('invalid parser specified: {}'.format(exception))
 
 
-def validate_resources(resources, ctx):
+def validate_resources(resources, ctx):  # pylint: disable=unused-argument
     """Validate the resources.
 
     :raises InputValidationError: if `num_machines` is not specified or is not an integer.

aiida/manage/caching.py

@@ -22,7 +22,7 @@
 from aiida.common import exceptions
 from aiida.common.lang import type_check
 
-from aiida.plugins.entry_point import ENTRY_POINT_STRING_SEPARATOR, entry_point_group_to_module_path_map
+from aiida.plugins.entry_point import ENTRY_POINT_STRING_SEPARATOR, ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP
 
 __all__ = ('get_use_cache', 'enable_caching', 'disable_caching')
 
@@ -248,7 +248,7 @@ def _validate_identifier_pattern(*, identifier):
 
     1.  <group_name><ENTRY_POINT_STRING_SEPARATOR><tail>
 
-        where `group_name` is one of the keys in `entry_point_group_to_module_path_map`
+        where `group_name` is one of the keys in `ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP`
         and `tail` can be anything _except_ `ENTRY_POINT_STRING_SEPARATOR`.
 
     2. a fully qualified Python name
@@ -276,7 +276,7 @@ def _validate_identifier_pattern(*, identifier):
         group_pattern, _ = identifier.split(ENTRY_POINT_STRING_SEPARATOR)
         if not any(
             _match_wildcard(string=group_name, pattern=group_pattern)
-            for group_name in entry_point_group_to_module_path_map
+            for group_name in ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP
         ):
             raise ValueError(
                 common_error_msg + "Group name pattern '{}' does not match any of the AiiDA entry point group names.".
@@ -290,7 +290,7 @@ def _validate_identifier_pattern(*, identifier):
     # aiida.* or aiida.calculations*
     if '*' in identifier:
         group_part, _ = identifier.split('*', 1)
-        if any(group_name.startswith(group_part) for group_name in entry_point_group_to_module_path_map):
+        if any(group_name.startswith(group_part) for group_name in ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP):
             return
     # Finally, check if it could be a fully qualified Python name
     for identifier_part in identifier.split('.'):