test turbo server

.github/workflows/eco-vcenter-ci.yaml

@@ -44,3 +44,9 @@ jobs:
         working-directory: ansible_collections/vmware/vmware
         env:
           ANSIBLE_COLLECTIONS_PATH: "${{ github.workspace }}"
+
+      - name: kill turbo server
+        if: ${{ always() }}
+        run: |
+          # kill turbo server
+          ps -ef | grep turbo | grep -v 'grep' |awk '{print $2}' | xargs kill -9

docs/module_profiling.md

@@ -0,0 +1,46 @@
+# Module Profiling and Performance
+
+It might be useful to check module performance when developing.
+
+## Timing Tasks
+
+Ansible comes with a basic profiler that outputs how long each task takes. You can enable it by setting an environment variable:
+```bash
+export ANSIBLE_CALLBACKS_ENABLED=profile_tasks
+```
+
+## Profiling Modules
+
+Python comes with a profiler that will show how long each method takes. Heres a very basic example:
+```python
+def main():
+    # import the profiler and setup the context manager
+    import cProfile
+    import pstats
+    with cProfile.Profile() as pr:
+        argument_spec = VmwareRestClient.vmware_client_argument_spec()
+        module = AnsibleModule(
+            argument_spec=argument_spec,
+            supports_check_mode=True,
+        )
+
+        # Execute your module as usual
+        my_class = Foo(module)
+        out = my_class.do()
+
+        # save the profile results to a variable
+        stats = pstats.Stats(pr)
+
+    # sort by time and write the output to a file so you can view the results
+    stats.sort_stats(pstats.SortKey.TIME)
+    stats.dump_stats("/some/local/path/profile.prof")
+
+    # exit module as usual
+    module.exit_json(changed=False, out=out)
+```
+
+You can use a tool like `snakeviz` to view the profile results.
+```bash
+pip install snakeviz
+snakviz /some/local/path/profile.prof
+```

docs/turbo_server.md

@@ -0,0 +1,95 @@
+# Using the Turbo Server and Cache
+
+The VMware modules make a lot of API calls that can take a (relatively) long time to finish. To help speed things up, you can enable some caching functionality.
+
+The cache relies on the turbo server provided by `cloud.common`. (ADD LINK HERE)
+
+## Enabling and Configuring
+
+To enable the cache, set the environment variable:
+```bash
+export ENABLE_TURBO_SERVER=1
+```
+
+The cache expires every 15 seconds by default. To change the length of time before the cache expires, set the environment variable:
+```bash
+export ANSIBLE_TURBO_LOOKUP_TTL=120
+```
+
+You can also set these variables in your playbook, if that's more convenient:
+```yaml
+- name: Example
+  hosts: localhost
+  environment:
+    ENABLE_TURBO_SERVER: 1
+    ANSIBLE_TURBO_LOOKUP_TTL: 120
+  tasks:
+  ...
+```
+
+### Clearing The Cache
+
+You may find the need to clear the cache manually. This will make sure that all cached method return values are invalidated. You can do so with the `clear_cache` module:
+```yaml
+- name: Clear the cache
+  vmware.vmware.clear_cache: {}
+```
+
+### Killing the turbo server
+
+You may want to kill the turbo server before its expriation time. This will clear the cache and also delete any cached module files. You can do so by terminating the process running on the remote host (the host that the vmware.vmware task was run on):
+```bash
+ps -ef | grep turbo | grep -v grep | awk '{print $2}' | xargs kill
+```
+
+## Development
+
+To use the turbo server in your module, you need to replace the AnsibleModule import with the custom class from this repo.
+```python
+from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import (
+    AnsibleModule
+)
+```
+
+You can leverage the cache from `functools` to save the results from a method. One use case would be caching an authentication session or the results from looking up VM information. To use the cache, import `functools` and then add the cache decorator to the method:
+```python
+import functools
+
+@functools.cache
+def my_method():
+    ....
+```
+
+When attaching the cache decorator to methods, the argument inputs are hashed and compared to previous calls to determine if a cached result can be used. This means if your method uses a class, the class must have a reasonable hash and equals method defined. For example:
+```python
+class PyVmomi(object):
+    def __init__(self, module):
+        """
+        Constructor
+        """
+        self.module = module
+        self.params = module.params
+        ....
+
+    def __eq__(self, value):
+        if not isinstance(value, self.__class__):
+            return False
+        return bool(all([
+            (self.params['hostname'] == value.params['hostname']),
+            (self.params['username'] == value.params['username'])
+        ]))
+
+    def __hash__(self):
+        return hash(self.params['hostname'] + self.params['username'])
+```
+
+To clear the cache from within a module, you can call the builtin `clear_vmware_cache` method. If the user is not using the turbo server, the module does nothing so you can call this method safely without checking.
+```python
+def main():
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+    ....
+    module.clear_vmware_cache()
+```

galaxy.yml

@@ -22,3 +22,5 @@ build_ignore:
   - changelogs/.plugin-cache.yaml
   - .github
   - .vscode
+dependencies:
+  cloud.common: "*"

plugins/module_utils/_vmware.py

@@ -33,6 +33,9 @@
     HAS_PYVMOMI = False
 
 from ansible.module_utils.basic import env_fallback, missing_required_lib
+from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import (
+    cache,
+)
 
 
 class ApiAccessError(Exception):
@@ -80,6 +83,7 @@
     )
 
 
+@cache
 def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None,
                    port=None, validate_certs=None,
                    httpProxyHost=None, httpProxyPort=None):
@@ -203,11 +207,26 @@
         self.module = module
         self.params = module.params
         self.current_vm_obj = None
-        self.si, self.content = connect_to_api(self.module, return_si=True)
+        self.si, self.content = self._connect_to_vcenter()
         self.custom_field_mgr = []
         if self.content.customFieldsManager:  # not an ESXi
             self.custom_field_mgr = self.content.customFieldsManager.field
 
+    def __eq__(self, value):
+        if not isinstance(value, self.__class__):
+            return False
+        return bool(all([
+            (self.params['hostname'] == value.params['hostname']),
+            (self.params['username'] == value.params['username'])
+        ]))
+
+    def __hash__(self):
+        return hash(self.params['hostname'] + self.params['username'])
+
+    @cache
+    def _connect_to_vcenter(self):
+        return connect_to_api(self.module, return_si=True)
+
     def is_vcenter(self):
         """
         Check if given hostname is vCenter or ESXi host
@@ -282,6 +301,7 @@
         """
         return self.get_objs_by_name_or_moid([vim.dvs.DistributedVirtualPortgroup], portgroup)
 
+    @cache
     def get_vm_using_params(
             self, name_param='name', uuid_param='uuid', moid_param='moid', fail_on_missing=False,
             name_match_param='name_match', use_instance_uuid_param='use_instance_uuid'):

plugins/module_utils/_vmware_ansible_module.py

@@ -0,0 +1,82 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+import os
+import functools
+
+from ansible.module_utils.common.validation import check_type_bool
+from ansible.module_utils.common.text.converters import to_native
+
+
+enable_turbo_mode = check_type_bool(os.environ.get("ENABLE_TURBO_MODE", False))
+
+if enable_turbo_mode:
+    try:
+        from ansible_collections.cloud.common.plugins.module_utils.turbo.module import (
+            AnsibleTurboModule as BaseAnsibleModule,
+        )
+
+        BaseAnsibleModule.collection_name = "vmware.vmware"
+    except ImportError:
+        from ansible.module_utils.basic import AnsibleModule as BaseAnsibleModule
+else:
+    from ansible.module_utils.basic import AnsibleModule as BaseAnsibleModule
+
+
+CACHED_FUNCTION_REGISTRY = set()
+
+
+class AnsibleModule(BaseAnsibleModule):
+    """
+    The exit_json and __format_value_for_turbo_server should really be added to the upstream
+    cloud.common repo, but until then we need it here.
+    The outputs from a module need to be passed through the turbo server using pickle. If the output
+    contains something that pickle cannot encode/decode, we need to convert it first.
+    For most APIs that return content as JSON, this isn't an issue. But for the SDKs VMware uses,
+    it can be a problem.
+    """
+    def exit_json(self, **kwargs):
+        if enable_turbo_mode:
+            kwargs = self.__format_value_for_turbo_server(kwargs)
+        super().exit_json(**kwargs)
+
+    def __format_value_for_turbo_server(self, value):
+        if isinstance(value, (str, bool, int)):
+            return value
+        if isinstance(value, set):
+            return self.__format_value_for_turbo_server(list(value))
+        if isinstance(value, list):
+            return [self.__format_value_for_turbo_server(v) for v in value]
+        if isinstance(value, dict):
+            for k, v in value.items():
+                value[k] = self.__format_value_for_turbo_server(v)
+            return value
+
+        return to_native(value)
+
+    def clear_vmware_cache(self, funcs=None):
+        cleared = set()
+        no_cache = set()
+        if not funcs:
+            funcs = CACHED_FUNCTION_REGISTRY
+
+        for f in funcs:
+            try:
+                f.cache_clear()
+                cleared.add(f.__name__)
+            except AttributeError:
+                no_cache.add(f.__name__)
+
+        return cleared, no_cache
+
+
+def cache(func):
+    @functools.wraps(func)
+    @functools.lru_cache(maxsize=128)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    CACHED_FUNCTION_REGISTRY.add(wrapper)
+    return wrapper

plugins/module_utils/_vmware_facts.py

@@ -19,16 +19,29 @@
 except ImportError:
     pass
 
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.six import integer_types, string_types, iteritems
-import ansible.module_utils.common._collections_compat as collections_compat
+import ansible.module_utils.six.moves.collections_abc as collections_compat
 from ansible_collections.vmware.vmware.plugins.module_utils._vmware_folder_paths import get_folder_path_of_vm
+from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import (
+    cache,
+)
 
 
 class VmFacts():
     def __init__(self, vm):
         self.vm = vm
 
+    def __eq__(self, value):
+        if not isinstance(value, self.__class__):
+            return False
+        return bool(all([
+            (self.vm._GetMoId() == value.vm._GetMoId())
+        ]))
+
+    def __hash__(self):
+        return hash(self.vm._GetMoId())
+
     def hw_all_facts(self):
         '''
         Returns a combined set of all 'hw_' facts
@@ -42,6 +55,7 @@
             **self.hw_network_device_facts()
         }
 
+    @cache
     def all_facts(self, content):
         return {
             **self.hw_all_facts(),
@@ -355,7 +369,7 @@
         elif isinstance(xo, vim.vm.device.VirtualDisk):
             data[x] = serialize_spec(xo)
         elif isinstance(xo, vim.vm.device.VirtualDeviceSpec.FileOperation):
-            data[x] = to_text(xo)
+            data[x] = to_native(xo)
         elif isinstance(xo, vim.Description):
             data[x] = {
                 'dynamicProperty': serialize_spec(xo.dynamicProperty),
@@ -364,7 +378,7 @@
                 'summary': serialize_spec(xo.summary),
             }
         elif hasattr(xo, 'name'):
-            data[x] = to_text(xo) + ':' + to_text(xo.name)
+            data[x] = to_native(xo) + ':' + to_native(xo.name)
         elif isinstance(xo, vim.vm.ProfileSpec):
             pass
         elif issubclass(xt, list):
@@ -375,13 +389,13 @@
             if issubclass(xt, integer_types):
                 data[x] = int(xo)
             else:
-                data[x] = to_text(xo)
+                data[x] = to_native(xo)
         elif issubclass(xt, bool):
             data[x] = xo
         elif issubclass(xt, dict):
-            data[to_text(x)] = {}
+            data[to_native(x)] = {}
             for k, v in xo.items():
-                k = to_text(k)
+                k = to_native(k)
                 data[x][k] = serialize_spec(v)
         else:
             data[x] = str(xt)