Clean-up of the autogroup logic

In particular:

- deprecated use of 'group-name' in favour of 'group-label'
  both in the internal API of AutoGroups and in the CLI parameters
  of `verdi run`
- Improved the Autogroup implementation and API
- add various tests of the functionality of autogroups
- fixed the CLI parameters to include/exclude groups, that were badly
  broken (and untested). Now it accepts any entrypoint string (before
  it was accepting the first part of an node_type string). Also added
  click validation of the parameters.
- fixed the --group flag to activate/deactivate autogrouping (it was
  wrongly defined and thus always True)
- linter fixes

.pre-commit-config.yaml

@@ -53,7 +53,6 @@
         aiida/common/datastructures.py|
         aiida/engine/daemon/execmanager.py|
         aiida/engine/processes/calcjobs/tasks.py|
-        aiida/orm/autogroup.py|
         aiida/orm/querybuilder.py|
         aiida/orm/nodes/data/array/bands.py|
         aiida/orm/nodes/data/array/projection.py|
@@ -66,7 +65,6 @@
         aiida/parsers/plugins/arithmetic/add.py|
         aiida/parsers/plugins/templatereplacer/doubler.py|
         aiida/parsers/plugins/templatereplacer/__init__.py|
-        aiida/plugins/entry_point.py|
         aiida/plugins/entry.py|
         aiida/plugins/info.py|
         aiida/plugins/registry.py|

aiida/cmdline/commands/cmd_plugin.py

@@ -13,7 +13,7 @@
 
 from aiida.cmdline.commands.cmd_verdi import verdi
 from aiida.cmdline.utils import decorators, echo
-from aiida.plugins.entry_point import entry_point_group_to_module_path_map
+from aiida.plugins.entry_point import ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP
 
 
 @verdi.group('plugin')
@@ -22,7 +22,7 @@ def verdi_plugin():
 
 
 @verdi_plugin.command('list')
-@click.argument('entry_point_group', type=click.Choice(entry_point_group_to_module_path_map.keys()), required=False)
+@click.argument('entry_point_group', type=click.Choice(ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP.keys()), required=False)
 @click.argument('entry_point', type=click.STRING, required=False)
 @decorators.with_dbenv()
 def plugin_list(entry_point_group, entry_point):
@@ -34,7 +34,7 @@ def plugin_list(entry_point_group, entry_point):
 
     if entry_point_group is None:
         echo.echo_info('Available entry point groups:')
-        for group in sorted(entry_point_group_to_module_path_map.keys()):
+        for group in sorted(ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP.keys()):
             echo.echo('* {}'.format(group))
 
         echo.echo('')

aiida/cmdline/commands/cmd_run.py

@@ -10,13 +10,17 @@
 """`verdi run` command."""
 import contextlib
 import os
+import functools
 import sys
+import warnings
 
 import click
 
 from aiida.cmdline.commands.cmd_verdi import verdi
 from aiida.cmdline.params.options.multivalue import MultipleValueOption
 from aiida.cmdline.utils import decorators, echo
+from aiida.common.warnings import AiidaDeprecationWarning
+from aiida.orm import autogroup
 
 
 @contextlib.contextmanager
@@ -37,35 +41,65 @@ def update_environment(argv):
         sys.path = _path
 
 
+def validate_entrypoint_string_or_all(ctx, param, value, allow_all=True):  # pylint: disable=unused-argument,invalid-name
+    """Validate that `value` is a valid entrypoint string or the string 'all'."""
+    try:
+        autogroup.Autogroup.validate(value, allow_all=allow_all)
+    except Exception as exc:
+        raise click.BadParameter(str(exc) + ' ({})'.format(value))
+
+    return value
+
+
 @verdi.command('run', context_settings=dict(ignore_unknown_options=True,))
 @click.argument('scriptname', type=click.STRING)
 @click.argument('varargs', nargs=-1, type=click.UNPROCESSED)
-@click.option('-g', '--group', is_flag=True, default=True, show_default=True, help='Enables the autogrouping')
-@click.option('-n', '--group-name', type=click.STRING, required=False, help='Specify the name of the auto group')
-@click.option('-e', '--exclude', cls=MultipleValueOption, default=[], help='Exclude these classes from auto grouping')
+@click.option('--group/--no-group', default=True, show_default=True, help='Enables the autogrouping')
+@click.option('-l', '--group-label', type=click.STRING, required=False, help='Specify the label of the auto group')
 @click.option(
-    '-i', '--include', cls=MultipleValueOption, default=['all'], help='Include these classes from auto grouping'
+    '-n',
+    '--group-name',
+    type=click.STRING,
+    required=False,
+    help='Specify the name of the auto group [DEPRECATED, USE --group-label instead]'
+)
+@click.option(
+    '-e',
+    '--exclude',
+    cls=MultipleValueOption,
+    default=[],
+    help='Exclude these classes from auto grouping (use full entrypoint strings)',
+    callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
+)
+@click.option(
+    '-i',
+    '--include',
+    cls=MultipleValueOption,
+    default=['all'],
+    help='Include these classes from auto grouping  (use full entrypoint strings or "all")',
+    callback=validate_entrypoint_string_or_all
 )
 @click.option(
     '-E',
     '--excludesubclasses',
     cls=MultipleValueOption,
     default=[],
-    help='Exclude these classes and their sub classes from auto grouping'
+    help='Exclude these classes and their sub classes from auto grouping (use full entrypoint strings)',
+    callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
 )
 @click.option(
     '-I',
     '--includesubclasses',
     cls=MultipleValueOption,
     default=[],
-    help='Include these classes and their sub classes from auto grouping'
+    help='Include these classes and their sub classes from auto grouping (use full entrypoint strings)',
+    callback=functools.partial(validate_entrypoint_string_or_all, allow_all=False)
 )
 @decorators.with_dbenv()
-def run(scriptname, varargs, group, group_name, exclude, excludesubclasses, include, includesubclasses):
+def run(scriptname, varargs, group, group_label, group_name, exclude, excludesubclasses, include, includesubclasses):
     # pylint: disable=too-many-arguments,exec-used
     """Execute scripts with preloaded AiiDA environment."""
     from aiida.cmdline.utils.shell import DEFAULT_MODULES_LIST
-    from aiida.orm import autogroup
 
     # Prepare the environment for the script to be run
     globals_dict = {
@@ -80,22 +114,25 @@ def run(scriptname, varargs, group, group_name, exclude, excludesubclasses, incl
     for app_mod, model_name, alias in DEFAULT_MODULES_LIST:
         globals_dict['{}'.format(alias)] = getattr(__import__(app_mod, {}, {}, model_name), model_name)
 
-    if group:
-        automatic_group_name = group_name
-        if automatic_group_name is None:
-            from aiida.common import timezone
+    if group_name:
+        warnings.warn('--group-name is deprecated, use `--group-label` instead', AiidaDeprecationWarning)  # pylint: disable=no-member
+        if group_label:
+            raise click.BadParameter('You cannot specify both --group-name and --group-label; use --group-label only')
+        group_label = group_name
 
-            automatic_group_name = 'Verdi autogroup on ' + timezone.now().strftime('%Y-%m-%d %H:%M:%S')
+    if group:
+        automatic_group_label = group_label
 
         aiida_verdilib_autogroup = autogroup.Autogroup()
+        if automatic_group_label is not None:
+            aiida_verdilib_autogroup.set_group_label(automatic_group_label)
         aiida_verdilib_autogroup.set_exclude(exclude)
         aiida_verdilib_autogroup.set_include(include)
         aiida_verdilib_autogroup.set_exclude_with_subclasses(excludesubclasses)
         aiida_verdilib_autogroup.set_include_with_subclasses(includesubclasses)
-        aiida_verdilib_autogroup.set_group_name(automatic_group_name)
 
         # Note: this is also set in the exec environment! This is the intended behavior
-        autogroup.current_autogroup = aiida_verdilib_autogroup
+        autogroup.CURRENT_AUTOGROUP = aiida_verdilib_autogroup
 
     # Initialize the variable here, otherwise we get UnboundLocalError in the finally clause if it fails to open
     handle = None