GroupPath: a utility to work with virtual Group hierarchies (#3613)

Groups can be used to store nodes in AiiDA, but do not have any builtin
hierarchy themselves. However, often it may be useful to think of groups
as folders on a filesystem and the nodes within them as the files.

Building this functionality directly on the database would require
significant changes, but a virtual hierarchy based on the group labels
can be readily provided. This is what the new utility class `GroupPath`
facilitates. It allows group labels to be interpreted as the hierarchy
of groups. Example: consider one has groups with the following labels

    group/sub/a
    group/sub/b
    group/other/c

One could see this as the group `group` containing the sub groups `sub`
and `other`, with `sub` containing `a` and `b` itself. The `GroupPath`
class allows one to exploit this hierarchical naming::

    path = GroupPath('group')
    path.sub.a.get_group()  # will return group with label `group/sub/a`

It can also be used to create groups that do not yet exist:

    path = GroupPath()
    path.some.group.get_or_create_group()

This will create a `Group` with the label `some/group`. The `GroupPath`
class implements many other useful methods to make the traversing and
manipulating of groups a lot easier.

.ci/workchains.py

@@ -7,6 +7,7 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
+# pylint: disable=invalid-name
 from aiida.common import AttributeDict
 from aiida.engine import calcfunction, workfunction, WorkChain, ToContext, append_, while_, ExitCode
 from aiida.engine import BaseRestartWorkChain, process_handler, ProcessHandlerReport

.pylintrc

@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=bad-continuation,locally-disabled,useless-suppression,django-not-available,bad-option-value,logging-format-interpolation,no-else-raise,import-outside-toplevel
+disable=bad-continuation,locally-disabled,useless-suppression,django-not-available,bad-option-value,logging-format-interpolation,no-else-raise,import-outside-toplevel,cyclic-import
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

aiida/cmdline/commands/cmd_group.py

@@ -361,3 +361,72 @@ def group_copy(source_group, destination_group):
     # Copy nodes
     dest_group.add_nodes(list(source_group.nodes))
     echo.echo_success('Nodes copied from group<{}> to group<{}>'.format(source_group.label, dest_group.label))
+
+
+@verdi_group.group('path')
+def verdi_group_path():
+    """Inspect groups of nodes, with delimited label paths."""
+
+
+@verdi_group_path.command('ls')
+@click.argument('path', type=click.STRING, required=False)
+@click.option('-R', '--recursive', is_flag=True, default=False, help='Recursively list sub-paths encountered')
+@click.option('-l', '--long', 'as_table', is_flag=True, default=False, help='List as a table, with sub-group count')
+@click.option(
+    '-d', '--with-description', 'with_description', is_flag=True, default=False, help='Show also the group description'
+)
+@click.option(
+    '--no-virtual',
+    'no_virtual',
+    is_flag=True,
+    default=False,
+    help='Only show paths that fully correspond to an existing group'
+)
+@click.option(
+    '-t',
+    '--type',
+    'group_type',
+    type=types.LazyChoice(valid_group_type_strings),
+    default=user_defined_group,
+    help='Show groups of a specific type, instead of user-defined groups. Start with semicolumn if you want to '
+    'specify aiida-internal type'
+)
+@click.option('--no-warn', is_flag=True, default=False, help='Do not issue a warning if any paths are invalid.')
+@with_dbenv()
+def group_path_ls(path, recursive, as_table, no_virtual, group_type, with_description, no_warn):
+    # pylint: disable=too-many-arguments
+    """Show a list of existing group paths."""
+    from aiida.tools.groups.paths import GroupPath, InvalidPath
+
+    try:
+        path = GroupPath(path or '', type_string=group_type, warn_invalid_child=not no_warn)
+    except InvalidPath as err:
+        echo.echo_critical(str(err))
+
+    if recursive:
+        children = path.walk()
+    else:
+        children = path.children
+
+    if as_table or with_description:
+        from tabulate import tabulate
+        headers = ['Path', 'Sub-Groups']
+        if with_description:
+            headers.append('Description')
+        rows = []
+        for child in sorted(children):
+            if no_virtual and child.is_virtual:
+                continue
+            row = [
+                child.path if child.is_virtual else click.style(child.path, bold=True),
+                len([c for c in child.walk() if not c.is_virtual])
+            ]
+            if with_description:
+                row.append('-' if child.is_virtual else child.get_group().description)
+            rows.append(row)
+        echo.echo(tabulate(rows, headers=headers))
+    else:
+        for child in sorted(children):
+            if no_virtual and child.is_virtual:
+                continue
+            echo.echo(child.path, bold=not child.is_virtual)

aiida/tools/groups/__init__.py

@@ -0,0 +1,11 @@
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=wildcard-import,undefined-variable
+"""Provides tools for interacting with AiiDA Groups."""
+from .paths import *
+
+__all__ = paths.__all__