Merge branch 'master' into feature/3.21.0

Makefile

@@ -68,6 +68,7 @@ coverage: ## check code coverage quickly with the default Python
 docs: ## generate Sphinx HTML documentation
 	$(MAKE) -C docs clean
 	cci task doc > docs/tasks.rst
+	cci flow doc > docs/flows.rst
 	$(MAKE) -C docs html
 	$(BROWSER) docs/_build/html/index.html
 

cumulusci/cli/cci.py

@@ -46,13 +46,14 @@
 from cumulusci.cli.runtime import get_installed_version
 from cumulusci.cli.ui import CliTable, CROSSMARK, SimpleSalesforceUIHelpers
 from cumulusci.salesforce_api.utils import get_simple_salesforce_connection
-from cumulusci.utils import doc_task
+from cumulusci.utils import doc_task, document_flow, flow_ref_title_and_intro
 from cumulusci.utils import parse_api_datetime
 from cumulusci.utils import get_cci_upgrade_command
 from cumulusci.utils.git import current_branch
 from cumulusci.utils.logging import tee_stdout_stderr
 from cumulusci.oauth.salesforce import CaptureSalesforceOAuth
 from cumulusci.core.utils import cleanup_org_cache_dirs
+from cumulusci.utils.yaml.cumulusci_yml import cci_safe_load
 
 
 from .logger import init_logger, get_tempfile_logger
@@ -1360,6 +1361,56 @@ def task_doc(runtime):
         click.echo("")
 
 
+@flow.command(name="doc", help="Exports RST format documentation for all flows")
+@pass_runtime(require_project=False)
+def flow_doc(runtime):
+    with open("docs/flows.yml", "r", encoding="utf-8") as f:
+        flow_info = cci_safe_load(f)
+
+    click.echo(flow_ref_title_and_intro(flow_info["intro_blurb"]))
+
+    flow_info_groups = list(flow_info["groups"].keys())
+
+    flows = (
+        runtime.project_config.list_flows()
+        if runtime.project_config is not None
+        else runtime.universal_config.list_flows()
+    )
+    flows_by_group = group_items(flows)
+    flow_groups = sorted(
+        flows_by_group.keys(),
+        key=lambda group: flow_info_groups.index(group)
+        if group in flow_info_groups
+        else 100,
+    )
+
+    for group in flow_groups:
+        click.echo(f"{group}\n{'-' * len(group)}")
+        if group in flow_info["groups"]:
+            click.echo(flow_info["groups"][group]["description"])
+
+        for flow in sorted(flows_by_group[group]):
+            flow_name, flow_description = flow
+            try:
+                flow_coordinator = runtime.get_flow(flow_name)
+            except FlowNotFoundError as e:
+                raise click.UsageError(str(e))
+
+            additional_info = None
+            if flow_name in flow_info.get("flows", {}):
+                additional_info = flow_info["flows"][flow_name]["rst_text"]
+
+            click.echo(
+                document_flow(
+                    flow_name,
+                    flow_description,
+                    flow_coordinator,
+                    additional_info=additional_info,
+                )
+            )
+            click.echo("")
+
+
 @task.command(name="info", help="Displays information for a task")
 @click.argument("task_name")
 @pass_runtime(require_project=False, require_keychain=True)

cumulusci/cli/tests/test_cci.py

@@ -1956,6 +1956,58 @@ def test_flow_info__not_found(self):
         with self.assertRaises(click.UsageError):
             run_click_command(cci.flow_info, runtime=runtime, flow_name="test")
 
+    @mock.patch("cumulusci.cli.cci.group_items")
+    @mock.patch("cumulusci.cli.cci.document_flow")
+    def test_flow_doc__no_flows_rst_file(self, doc_flow, group_items):
+        runtime = mock.Mock()
+        runtime.universal_config.flows = {"test": {}}
+        flow_config = FlowConfig({"description": "Test Flow", "steps": {}})
+        runtime.get_flow.return_value = FlowCoordinator(None, flow_config)
+
+        group_items.return_value = {"Group One": [["test flow", "description"]]}
+
+        run_click_command(cci.flow_doc, runtime=runtime)
+        group_items.assert_called_once()
+        doc_flow.assert_called()
+
+    @mock.patch("cumulusci.cli.cci.click.echo")
+    @mock.patch("cumulusci.cli.cci.cci_safe_load")
+    def test_flow_doc__with_flows_rst_file(self, safe_load, echo):
+        runtime = CliRuntime(
+            config={
+                "flows": {
+                    "Flow1": {
+                        "steps": {},
+                        "description": "Description of Flow1",
+                        "group": "Group1",
+                    }
+                }
+            },
+            load_keychain=False,
+        )
+
+        safe_load.return_value = {
+            "intro_blurb": "opening blurb for flow reference doc",
+            "groups": {
+                "Group1": {"description": "This is a description of group1."},
+            },
+            "flows": {"Flow1": {"rst_text": "Some ``extra`` **pizzaz**!"}},
+        }
+
+        run_click_command(cci.flow_doc, runtime=runtime)
+
+        assert 1 == safe_load.call_count
+
+        expected_call_args = [
+            "Flow Reference\n==========================================\n\nopening blurb for flow reference doc\n.. contents::\n    :depth: 2\n    :local:\n\n",
+            "Group1\n------",
+            "This is a description of group1.",
+            "Flow1\n^^^^^\n\n**Description:** Description of Flow1\n\nSome ``extra`` **pizzaz**!\n**Flow Steps**\n\n.. code-block:: console\n",
+            "",
+        ]
+        expected_call_args = [mock.call(s) for s in expected_call_args]
+        assert echo.call_args_list == expected_call_args
+
     def test_flow_run(self):
         org_config = mock.Mock(scratch=True, config={})
         runtime = CliRuntime(
@@ -2009,9 +2061,7 @@ def test_flow_run_o_error(self):
             )
         assert "-o" in str(e.value)
 
-    def test_flow_run_delete_non_scratch(
-        self,
-    ):
+    def test_flow_run_delete_non_scratch(self):
         org_config = mock.Mock(scratch=False)
         runtime = mock.Mock()
         runtime.get_org.return_value = ("test", org_config)

cumulusci/core/flowrunner.py

@@ -283,12 +283,24 @@ def _rule(self, fill="=", length=60, new_line=False):
             self.logger.info("")
 
     def get_summary(self):
-        """Returns an output string that contains the description of the flow,
-        the sequence of tasks and subflows, and any "when" conditions associated
-        with tasks."""
+        """Returns an output string that contains the description of the flow
+        and its steps."""
         lines = []
         if "description" in self.flow_config.config:
             lines.append(f"Description: {self.flow_config.config['description']}")
+
+        step_lines = self.get_flow_steps()
+        if step_lines:
+            lines.append("\nFlow Steps")
+        lines.extend(step_lines)
+
+        return "\n".join(lines)
+
+    def get_flow_steps(self, for_docs=False):
+        """Returns a list of flow steps (tasks and sub-flows) for the given flow.
+        For docs, indicates whether or not we want to use the string for use in a code-block
+        of an rst file. If True, will omit output of source information."""
+        lines = []
         previous_parts = []
         previous_source = None
         for step in self.steps:
@@ -311,18 +323,29 @@ def get_summary(self):
                 else:
                     source = ""
                 if len(previous_parts) < i + 1 or previous_parts[i] != flow_name:
+                    if for_docs:
+                        source = ""
+
                     lines.append(f"{'    ' * i}{steps[i]}) flow: {flow_name}{source}")
                     if source:
                         new_source = ""
 
             padding = "    " * (i + 1) + " " * len(str(steps[i + 1]))
-            when = f"\n{padding}  when: {step.when}" if step.when is not None else ""
+            when = f"{padding}  when: {step.when}" if step.when is not None else ""
+
+            if for_docs:
+                new_source = ""
+
             lines.append(
-                f"{'    ' * (i + 1)}{steps[i + 1]}) task: {task_name}{new_source}{when}"
+                f"{'    ' * (i + 1)}{steps[i + 1]}) task: {task_name}{new_source}"
             )
+            if when:
+                lines.append(when)
+
             previous_parts = parts
             previous_source = step.project_config.source
-        return "\n".join(lines)
+
+        return lines
 
     def run(self, org_config):
         self.org_config = org_config

cumulusci/core/tests/test_flowrunner.py

@@ -150,13 +150,46 @@ def test_get_summary(self):
         actual_output = flow.get_summary()
         expected_output = (
             "Description: test description"
+            + "\n\nFlow Steps"
             + "\n1) flow: nested_flow_2 [from current folder]"
             + "\n    1) task: pass_name"
             + "\n    2) flow: nested_flow"
             + "\n        1) task: pass_name"
         )
         self.assertEqual(expected_output, actual_output)
 
+    def test_get_flow_steps(self):
+        self.project_config.config["flows"]["test"] = {
+            "description": "test description",
+            "steps": {"1": {"flow": "nested_flow_2"}},
+        }
+        flow_config = self.project_config.get_flow("test")
+        flow = FlowCoordinator(self.project_config, flow_config, name="test_flow")
+        actual_output = flow.get_flow_steps()
+        expected_output = [
+            "1) flow: nested_flow_2 [from current folder]",
+            "    1) task: pass_name",
+            "    2) flow: nested_flow",
+            "        1) task: pass_name",
+        ]
+        self.assertEqual(expected_output, actual_output)
+
+    def test_get_flow_steps__for_docs(self):
+        self.project_config.config["flows"]["test"] = {
+            "description": "test description",
+            "steps": {"1": {"flow": "nested_flow_2"}},
+        }
+        flow_config = self.project_config.get_flow("test")
+        flow = FlowCoordinator(self.project_config, flow_config, name="test_flow")
+        actual_output = flow.get_flow_steps(for_docs=True)
+        expected_output = [
+            "1) flow: nested_flow_2",
+            "    1) task: pass_name",
+            "    2) flow: nested_flow",
+            "        1) task: pass_name",
+        ]
+        self.assertEqual(expected_output, actual_output)
+
     def test_get_summary__substeps(self):
         flow = FlowCoordinator.from_steps(
             self.project_config,
@@ -183,11 +216,13 @@ def test_get_summary__multiple_sources(self):
                 ),
             ],
         )
+        actual_output = flow.get_summary()
         assert (
-            "1) flow: test"
+            "\nFlow Steps"
+            + "\n1) flow: test"
             + "\n    1) task: other:test1 [from other source]"
             + "\n    2) task: test2 [from current folder]"
-        ) == flow.get_summary()
+        ) == actual_output
 
     def test_init__options(self):
         """ A flow can accept task options and pass them to the task. """

cumulusci/tests/test_utils.py

@@ -12,8 +12,10 @@
 import responses
 
 from cumulusci import utils
-from cumulusci.core.config import TaskConfig
+from cumulusci.core.config import TaskConfig, FlowConfig
+from cumulusci.core.flowrunner import FlowCoordinator
 from cumulusci.core.tasks import BaseTask
+from cumulusci.tests.util import create_project_config
 
 
 class FunTestTask(BaseTask):
@@ -261,6 +263,45 @@ def test_create_task_options_doc(self, option_info):
 
         assert option_two_doc == ["\t *Optional*", "\n\t Brief description here."]
 
+    def test_document_flow(self):
+        project_config = create_project_config("TestOwner", "TestRepo")
+        flow_config = FlowConfig({"description": "Test Flow", "steps": {}})
+        coordinator = FlowCoordinator(project_config, flow_config, name="test_flow")
+        flow_doc = utils.document_flow("test flow", "test description.", coordinator)
+
+        expected_doc = (
+            "test flow"
+            "\n^^^^^^^^^\n"
+            "\n**Description:** test description.\n"
+            "\n**Flow Steps**\n"
+            "\n.. code-block:: console\n"
+        )
+
+        assert expected_doc == flow_doc
+
+    def test_document_flow__additional_info(self):
+        flow_steps = ["1) (Task) Extract"]
+        flow_coordinator = mock.Mock(get_flow_steps=mock.Mock(return_value=flow_steps))
+        other_info = "**this is** just some rst ``formatted`` text."
+
+        flow_doc = utils.document_flow(
+            "test flow",
+            "test description.",
+            flow_coordinator,
+            additional_info=other_info,
+        )
+
+        expected_doc = (
+            "test flow"
+            "\n^^^^^^^^^\n"
+            "\n**Description:** test description.\n"
+            f"\n{other_info}"
+            "\n**Flow Steps**\n"
+            "\n.. code-block:: console\n"
+            "\n\t1) (Task) Extract"
+        )
+        assert expected_doc == flow_doc
+
     @responses.activate
     def test_download_extract_zip(self):
         f = io.BytesIO()

cumulusci/utils/__init__.py

@@ -479,6 +479,43 @@ def create_task_options_doc(task_options):
     return doc
 
 
+def flow_ref_title_and_intro(intro_blurb):
+    return f"""Flow Reference
+==========================================
+\n{intro_blurb}
+.. contents::
+    :depth: 2
+    :local:
+
+"""
+
+
+def document_flow(flow_name, description, flow_coordinator, additional_info=None):
+    """Document (project specific) flow configurations in RST format"""
+    doc = []
+
+    doc.append(f"{flow_name}\n{'^' * len(flow_name)}\n")
+    doc.append(f"**Description:** {description}\n")
+
+    if additional_info:
+        doc.append(additional_info)
+
+    doc.append("**Flow Steps**\n")
+    doc.append(".. code-block:: console\n")
+    flow_step_lines = flow_coordinator.get_flow_steps(for_docs=True)
+    # extra indent beneath code-block and finish with pipe for extra space afterwards
+    flow_step_lines = [f"\t{line}" for line in flow_step_lines]
+    # fix when clauses
+    lines = []
+    for line in flow_step_lines:
+        if line.startswith("when"):
+            line = f"\t\t{line}"
+        lines.append(line)
+    doc.extend(lines)
+
+    return "\n".join(doc)
+
+
 def package_xml_from_dict(items, api_version, package_name=None):
     lines = []