stackstorm faster with-items

stackstorm_withitems/README.md

@@ -0,0 +1,61 @@
+# withitems Integration Pack
+
+StackStorm integration pack for faster `with-items` in orquesta workflows
+
+## Quick Start
+
+Run the following commands to install this pack and the install  on your StackStorm host:
+
+``` shell
+st2 pack install withitems
+st2 pack configure withitems
+```
+
+## Configuration
+
+Copy the example configuration in [withitems.example.yaml](./withitems.example.yaml)
+to `/opt/stackstorm/configs/withitems.yaml` and edit as required.
+
+* `st2api_key`- optional key for preventing token timeouts
+* `st2apiurl` - url for api
+* `st2authurl` - url for authentication
+* `st2baseurl` - url for base stackstorm api
+
+### Configuration Example
+
+The configuration below is an example of what a end-user config might look like.
+``` yaml
+st2api_key: null
+st2apiurl: https://localhost:443/api
+st2authurl: https://localhost:443/auth
+st2baseurl: https://localhost:443
+```
+
+## Actions
+
+* `withitems.with_items`
+
+
+### Action workflow Example - withitems.with_items
+with_items will be used inside a workflow.  Here is an example of that. The YAQL select function is very handy to format a list of objects to pass as a parameter.
+``` shell
+---    
+    version: 1.0 
+
+    description: A with items example
+
+    input:
+      - list_items
+
+    tasks:
+      task1:
+        action: withitems.with_items
+        input:
+          action: core.echo
+          parameters: "<% ctx().list_items.select({\"message\"=>concat(\"message \", $)}) %>"
+        next:
+          - when: <% succeeded() %>
+            do: task2
+      task2:
+        action: core.noop
+```

stackstorm_withitems/actions/with_items.py

@@ -0,0 +1,143 @@
+import copy
+import jinja2
+import re
+import os
+import six
+import time
+import urllib3
+
+from st2client.models import LiveAction
+from st2client.client import Client
+from st2client.commands import action as st2action
+from st2common.runners.base_action import Action
+
+
+JINJA_REGEXP = '({{(.*?)}})'
+
+class WithItemsAction(Action):
+
+    def __init__(self, config):
+        super(WithItemsAction, self).__init__(config)
+        self.jinja_env = jinja2.Environment()
+        self.jinja_pattern = re.compile(JINJA_REGEXP)
+        self.api_key = self.config.get("st2api_key", None)
+
+        if self.api_key is not None:
+            if os.environ.get("ST2_AUTH_TOKEN", None) is not None:
+                del os.environ["ST2_AUTH_TOKEN"]
+        self.client = Client(base_url=config["st2baseurl"],
+                auth_url=config["st2authurl"],
+                api_key=self.api_key,
+                api_url=config["st2apiurl"]
+                )
+
+    def unescape_jinja(self, expr):
+        if isinstance(expr, six.string_types):
+            return self.unescape_jinja_str(expr)
+        elif isinstance(expr, list):
+            return self.unescape_jinja_list(expr)
+        elif isinstance(expr, dict):
+            return self.unescape_jinja_dict(expr)
+        else:
+            raise TypeError("Unable to escape jinja expression for type: {var}".format(var=str(type(expr))))
+
+    def unescape_jinja_str(self, expr_str):
+        expr_str = expr_str.replace("{_{", "{{")
+        expr_str = expr_str.replace("}_}", "}}")
+        return expr_str
+
+    def unescape_jinja_list(self, expr_list):
+        return [self.unescape_jinja(expr) for expr in expr_list ]
+
+    def unescape_jinja_dict(self, expr_dict):
+        for k, v in six.iteritems(expr_dict):
+            expr_dict[k] = self.unescape_jinja(v)
+        return expr_dict
+
+    def render_jinja(self, context, expr):
+        if isinstance(expr, six.string_types):
+            return self.render_jinja_str(context, expr)
+        elif isinstance(expr, list):
+            return self.render_jinja_list(context, expr)
+        elif isinstance(expr, dict):
+            return self.render_jinja_dict(context, expr)
+        else:
+            raise TypeError("Unable to render Jinja expression for type: {}".format(type(expr)))
+
+    def render_jinja_str(self, context, expr_str):
+        # find all of the jinja patterns in expr_str
+        patterns = self.jinja_pattern.findall(expr_str)
+
+        # if the matched pattern matches the full expression, then pull out
+        # the first group [0][1] which is the content between the {{ }}
+        # then use this special rendering method
+        if patterns[0][0] == expr_str:
+            # we only have a single pattern, render it so that a native type
+            # will be returned
+            func = self.jinja_env.compile_expression(patterns[0][1], expr_str)
+            return func(**context)
+        else:
+            # we have multiple patterns in one string so rendering it
+            # "normallY" and this will return a string
+            template = self.jinja_env.from_string(expr_str)
+            return template.render(context)
+
+    def render_jinja_list(self, context, expr_list):
+        return [self.render_jinja(context, expr) for expr in expr_list]
+
+    def render_jinja_dict(self, context, expr_dict):
+        rendered = {}
+        for k, expr in six.iteritems(expr_dict):
+            rendered[k] = self.render_jinja(context, expr)
+        return rendered
+
+    def run(self, action, parameters, paging_limit, sleep_time, result=None, **kwargs):
+        # unescape jinja
+        result_expr = result
+        if result_expr:
+            result_expr = self.unescape_jinja(result_expr)
+
+        running_ids = []
+        finished = []
+        param_index = 0
+        success = True
+        while len(finished) < len(parameters):
+            # fill paging pool
+            while len(running_ids) < paging_limit and \
+                    len(parameters) != len(running_ids) + len(finished) : # end of list no more to page
+                self.logger.debug("creating action index: " + str(param_index))
+                execution = self.client.executions.create(
+                    LiveAction(action=action,
+                               parameters=parameters[param_index]))
+                running_ids.append(execution.id)
+                param_index+=1
+
+            # sleep and wait for some work to be done
+            time.sleep(sleep_time)
+
+            # check for completed or failures
+            execution_list = self.client.liveactions.query(id=",".join(running_ids))
+            for e_updated in execution_list:
+                if e_updated.status in st2action.LIVEACTION_COMPLETED_STATES:
+                    if e_updated.status != st2action.LIVEACTION_STATUS_SUCCEEDED:
+                        success = False
+                    self.logger.debug("finished execution: " + str(e_updated.id))
+                    running_ids.remove(e_updated.id)
+                    finished.append(e_updated)
+
+        # extract the information out of the output
+        outputs = []
+        for exe in finished:
+            outputs.append({"status": copy.deepcopy(exe.status),
+                            "result": copy.deepcopy(exe.__dict__["result"])})
+        results = []
+        if result_expr:
+            for output in outputs:
+                result = output["result"]
+                result_context = {"_": {"result": result}}
+                result = self.render_jinja(result_context, result_expr)
+                results.append(result)
+        else:
+            results = outputs
+
+        return (success, results)

stackstorm_withitems/actions/with_items.yaml

@@ -0,0 +1,29 @@
+---
+name: "with_items"
+runner_type: "python-script"
+description: "faster with items;"
+enabled: true
+entry_point: "with_items.py"
+parameters:
+  action:
+    type: string
+    required: true
+  parameters:
+    type: array
+    items:
+      type: object
+    description: "list of objects for parameters"
+  paging_limit:
+    type: integer
+    default: 5
+  sleep_time:
+    type: integer
+    default: 25
+  result:
+    type: string
+    description: "jinja string to {_{ _.result.xxx }_} portion of result to return"
+output_schema:
+  results:
+    type: "array"
+    items:
+      type: "object"

stackstorm_withitems/actions/with_sample.meta.yaml

@@ -0,0 +1,12 @@
+---
+name: "with_sample"
+runner_type: "orquesta"
+description: "sample with items workflow"
+enabled: true
+entry_point: workflows/with_sample.yaml
+parameters:
+  list_items:
+    type: array
+    items:
+      type: string 
+    default: ["1","2","3"]

stackstorm_withitems/actions/workflows/with_sample.yaml

@@ -0,0 +1,19 @@
+version: 1.0
+
+description: A with items example
+
+input:
+  - list_items
+
+tasks:
+  task1:
+    action: withitems.with_items
+    input:
+      action: core.echo
+      parameters: "<% ctx().list_items.select({\"message\"=>concat(\"message \", $)}) %>"
+    next:
+      - when: <% succeeded() %>
+        do: task2
+  task2:
+    action: core.noop
+

stackstorm_withitems/config.schema.yaml

@@ -0,0 +1,18 @@
+---
+  st2baseurl:
+    description: "base host url for st2 api"
+    type: "string"
+    default: "http://localhost"
+  st2authurl:
+    description: "auth url for st2 api"
+    type: "string"
+    default: "http://localhost/auth"
+  st2apiurl:
+    description: "api url for st2 api"
+    type: "string"
+    default: "http://localhost/api"
+  st2api_key:
+    description: "api key used by stackstorm actions create with_items"
+    type: string
+    secret: true
+    required: false

stackstorm_withitems/pack.yaml

@@ -0,0 +1,9 @@
+---
+ref: withitems 
+name: withitems 
+description: faster with items
+version: 0.1.0
+author: Aaron Jonen, Nick Maludy
+email: [email protected]
+python_versions:
+  - "3"