From 0bd1cd49e04500a7f7a273c5d582323de6561a43 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 14:49:11 -0400 Subject: [PATCH 01/17] test turbo server --- .../module_utils/_vmware_ansible_module.py | 25 +++++++++++++++++++ plugins/modules/appliance_info.py | 4 ++- plugins/modules/cluster.py | 4 ++- plugins/modules/cluster_dpm.py | 4 ++- plugins/modules/cluster_drs.py | 4 ++- .../modules/cluster_drs_recommendations.py | 4 ++- plugins/modules/cluster_vcls.py | 4 ++- plugins/modules/content_template.py | 4 ++- plugins/modules/folder_template_from_vm.py | 4 ++- plugins/modules/guest_info.py | 4 ++- plugins/modules/license_info.py | 4 ++- plugins/modules/vcsa_settings.py | 4 ++- .../modules/vm_list_group_by_clusters_info.py | 4 ++- plugins/modules/vm_portgroup_info.py | 4 ++- 14 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 plugins/module_utils/_vmware_ansible_module.py diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py new file mode 100644 index 00000000..0c231dda --- /dev/null +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +import os + +from ansible.module_utils.common.validation import check_type_bool + +try: + enable_turbo_mode = check_type_bool(os.environ.get("ENABLE_TURBO_MODE")) +except TypeError: + enable_turbo_mode = False + +if enable_turbo_mode: + try: + from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 + AnsibleTurboModule as AnsibleModule, + ) + + AnsibleModule.collection_name = "vmware.vmware" + except ImportError: + from ansible.module_utils.basic import AnsibleModule # noqa: F401 +else: + from ansible.module_utils.basic import AnsibleModule # noqa: F401 diff --git a/plugins/modules/appliance_info.py b/plugins/modules/appliance_info.py index 8fbfe650..fa3e7322 100644 --- a/plugins/modules/appliance_info.py +++ b/plugins/modules/appliance_info.py @@ -177,7 +177,9 @@ } ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient diff --git a/plugins/modules/cluster.py b/plugins/modules/cluster.py index df75a599..c6d1c49a 100644 --- a/plugins/modules/cluster.py +++ b/plugins/modules/cluster.py @@ -88,7 +88,9 @@ except ImportError: pass -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible.module_utils._text import to_native from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( diff --git a/plugins/modules/cluster_dpm.py b/plugins/modules/cluster_dpm.py index d713b2db..47c4075c 100644 --- a/plugins/modules/cluster_dpm.py +++ b/plugins/modules/cluster_dpm.py @@ -108,7 +108,9 @@ except ImportError: pass -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, vmware_argument_spec diff --git a/plugins/modules/cluster_drs.py b/plugins/modules/cluster_drs.py index 06f96d84..368657fb 100644 --- a/plugins/modules/cluster_drs.py +++ b/plugins/modules/cluster_drs.py @@ -134,7 +134,9 @@ except ImportError: pass -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, vmware_argument_spec diff --git a/plugins/modules/cluster_drs_recommendations.py b/plugins/modules/cluster_drs_recommendations.py index c2c01ed5..09ebadaa 100644 --- a/plugins/modules/cluster_drs_recommendations.py +++ b/plugins/modules/cluster_drs_recommendations.py @@ -94,7 +94,9 @@ pass from itertools import zip_longest -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, vmware_argument_spec diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index 1f0c45ec..4f549b6e 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -133,7 +133,9 @@ except ImportError: pass -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible.module_utils._text import to_native from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( diff --git a/plugins/modules/content_template.py b/plugins/modules/content_template.py index cd9aac71..493b20ca 100644 --- a/plugins/modules/content_template.py +++ b/plugins/modules/content_template.py @@ -106,7 +106,9 @@ } ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient from ansible.module_utils._text import to_native diff --git a/plugins/modules/folder_template_from_vm.py b/plugins/modules/folder_template_from_vm.py index ec4b6686..daf322d9 100644 --- a/plugins/modules/folder_template_from_vm.py +++ b/plugins/modules/folder_template_from_vm.py @@ -142,7 +142,9 @@ import traceback -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi, vmware_argument_spec from ansible_collections.vmware.vmware.plugins.module_utils._vmware_folder_paths import format_folder_path_as_vm_fq_path from ansible_collections.vmware.vmware.plugins.module_utils._vmware_tasks import RunningTaskMonitor, TaskError diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index 603f17f7..34b12759 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -169,7 +169,9 @@ ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient from ansible_collections.vmware.vmware.plugins.module_utils._vmware_facts import ( diff --git a/plugins/modules/license_info.py b/plugins/modules/license_info.py index 3aedeb0a..2738bc8d 100644 --- a/plugins/modules/license_info.py +++ b/plugins/modules/license_info.py @@ -43,7 +43,9 @@ - 143cc-0e942-b2955-3ea12-d006f ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi, vmware_argument_spec diff --git a/plugins/modules/vcsa_settings.py b/plugins/modules/vcsa_settings.py index 65d165d8..75b7b63c 100644 --- a/plugins/modules/vcsa_settings.py +++ b/plugins/modules/vcsa_settings.py @@ -240,7 +240,9 @@ } ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient diff --git a/plugins/modules/vm_list_group_by_clusters_info.py b/plugins/modules/vm_list_group_by_clusters_info.py index e8c18b51..500fe9d7 100644 --- a/plugins/modules/vm_list_group_by_clusters_info.py +++ b/plugins/modules/vm_list_group_by_clusters_info.py @@ -113,7 +113,9 @@ } ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient diff --git a/plugins/modules/vm_portgroup_info.py b/plugins/modules/vm_portgroup_info.py index e0c15e96..0853a358 100644 --- a/plugins/modules/vm_portgroup_info.py +++ b/plugins/modules/vm_portgroup_info.py @@ -64,7 +64,9 @@ } ''' -from ansible.module_utils.basic import AnsibleModule +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi from ansible_collections.vmware.vmware.plugins.module_utils import _vmware_network as vmware_network from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient From 891305a52e65a64fed742b14761f9ad16541709e Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 14:54:47 -0400 Subject: [PATCH 02/17] force turbo mode to check times --- plugins/module_utils/_vmware_ansible_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 0c231dda..53cbff3f 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -12,6 +12,8 @@ except TypeError: enable_turbo_mode = False +# short circuit to test +enable_turbo_mode = True if enable_turbo_mode: try: from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 From db41cc774eaaab3ceacd646119881fa018f419bc Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 16:03:17 -0400 Subject: [PATCH 03/17] stash changes --- plugins/modules/cluster_vcls.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index 4f549b6e..387f6cc1 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -136,7 +136,7 @@ from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( AnsibleModule, ) -from ansible.module_utils._text import to_native +from ansible.module_utils.common.text.converters import to_native from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, @@ -148,6 +148,7 @@ ) + class VMwareClusterVcls(PyVmomi): def __init__(self, module): super(VMwareClusterVcls, self).__init__(module) @@ -283,7 +284,16 @@ def main(): if not module.check_mode: results['reconfig_task_result'] = vmware_cluster_vcls.configure_vcls(ds_to_add, ds_to_remove) - module.exit_json(**results) + # try: + module.exit_json(**{k:to_native(v) for k,v in results.items()}) + # except Exception as e: + # module.exit_json( + # changed=results['changed'], + # allowed_datastores=str(e), + # added_datastores=to_native(results['added_datastores']), + # removed_datastores=to_native(results['removed_datastores']), + # reconfig_task_result=to_native(results['reconfig_task_result']), + # ) if __name__ == '__main__': From d8264689d2018448c1bf928240323237528874c5 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 17:27:10 -0400 Subject: [PATCH 04/17] testing --- plugins/module_utils/_vmware_ansible_module.py | 1 - plugins/modules/cluster_vcls.py | 4 ++-- tests/integration/targets/init.sh | 4 ++++ tests/integration/targets/runme.sh | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 53cbff3f..099e3db5 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -13,7 +13,6 @@ enable_turbo_mode = False # short circuit to test -enable_turbo_mode = True if enable_turbo_mode: try: from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index 387f6cc1..5647489e 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -136,7 +136,7 @@ from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( AnsibleModule, ) -from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.common.text.converters import to_native, jsonify from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, @@ -285,7 +285,7 @@ def main(): results['reconfig_task_result'] = vmware_cluster_vcls.configure_vcls(ds_to_add, ds_to_remove) # try: - module.exit_json(**{k:to_native(v) for k,v in results.items()}) + module.exit_json(**results) # except Exception as e: # module.exit_json( # changed=results['changed'], diff --git a/tests/integration/targets/init.sh b/tests/integration/targets/init.sh index 492fce9b..68653756 100644 --- a/tests/integration/targets/init.sh +++ b/tests/integration/targets/init.sh @@ -3,6 +3,10 @@ export DEFAULT_COLLECTIONS_PATH="$ANSIBLE_COLLECTIONS_PATH/ansible_collections" +export ANSIBLE_TURBO_LOOKUP_TTL=1 +export ENABLE_TURBO_MODE=1 + + # Check if the variable is already set (e.g., in CI) if [ -z "$ANSIBLE_COLLECTIONS_PATH" ]; then # If not, use base collections path diff --git a/tests/integration/targets/runme.sh b/tests/integration/targets/runme.sh index 1e465575..ae85ee92 100755 --- a/tests/integration/targets/runme.sh +++ b/tests/integration/targets/runme.sh @@ -14,4 +14,4 @@ if [[ -n "$ANSIBLE_TAGS" ]]; then else echo "ANSIBLE_TAGS is not set for Eco vCenter. Running on simulator." exec ansible-playbook run.yml --tags integration-ci -fi \ No newline at end of file +fi From c9d66afca771a95f9d7707b423e521eaafd95130 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 17:27:20 -0400 Subject: [PATCH 05/17] cleanup --- plugins/modules/cluster_vcls.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index 5647489e..b57c16de 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -136,7 +136,7 @@ from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( AnsibleModule, ) -from ansible.module_utils.common.text.converters import to_native, jsonify +from ansible.module_utils.common.text.converters import to_native from ansible_collections.vmware.vmware.plugins.module_utils._vmware import ( PyVmomi, @@ -284,16 +284,7 @@ def main(): if not module.check_mode: results['reconfig_task_result'] = vmware_cluster_vcls.configure_vcls(ds_to_add, ds_to_remove) - # try: module.exit_json(**results) - # except Exception as e: - # module.exit_json( - # changed=results['changed'], - # allowed_datastores=str(e), - # added_datastores=to_native(results['added_datastores']), - # removed_datastores=to_native(results['removed_datastores']), - # reconfig_task_result=to_native(results['reconfig_task_result']), - # ) if __name__ == '__main__': From 17a5c924b449bbcfff3ee1ac6a6ff2b2f560024c Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Tue, 29 Oct 2024 21:38:57 -0400 Subject: [PATCH 06/17] test fixes --- plugins/module_utils/_vmware_ansible_module.py | 1 - plugins/module_utils/_vmware_tasks.py | 8 ++++---- plugins/modules/cluster_vcls.py | 6 +++--- tests/integration/targets/init.sh | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 099e3db5..0c231dda 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -12,7 +12,6 @@ except TypeError: enable_turbo_mode = False -# short circuit to test if enable_turbo_mode: try: from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 diff --git a/plugins/module_utils/_vmware_tasks.py b/plugins/module_utils/_vmware_tasks.py index 03630c7a..9853b1f5 100644 --- a/plugins/module_utils/_vmware_tasks.py +++ b/plugins/module_utils/_vmware_tasks.py @@ -14,7 +14,7 @@ import traceback from random import randint -from ansible.module_utils._text import to_text +from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.six import raise_from PYVMOMI_IMP_ERR = None @@ -60,7 +60,7 @@ def wait_for_completion(self, max_backoff=64, timeout=3600, vm=None, answers=Non if self.is_task_finished_with_success(): out = { - 'completion_time': self.task.info.completeTime, + 'completion_time': to_native(self.task.info.completeTime), 'state': self.task.info.state, 'result': self.task.info.result, 'entity_name': self.task.info.entityName, @@ -103,7 +103,7 @@ def handle_vm_questions(self): """ if hasattr(self.vm, "runtime") and self.vm.runtime.question: if not self.answers: - raise TaskError("Unanswered VM question: '%s'" % to_text(self.vm.runtime.question.text)) + raise TaskError("Unanswered VM question: '%s'" % to_native(self.vm.runtime.question.text)) responses = self.format_vm_question_responses() self.send_vm_question_responses(self.vm, responses) @@ -154,4 +154,4 @@ def send_vm_question_responses(self, responses): try: self.vm.AnswerVM(response["id"], response["response_num"]) except Exception as e: - raise TaskError("Answer failed: %s" % to_text(e)) + raise TaskError("Answer failed: %s" % to_native(e)) diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index b57c16de..1c70bd58 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -276,11 +276,11 @@ def main(): vmware_cluster_vcls = VMwareClusterVcls(module) ds_to_add, ds_to_remove, new_allowed_datastores = vmware_cluster_vcls.resolve_datastores_to_add_and_remove() - results['allowed_datastores'] = new_allowed_datastores + results['allowed_datastores'] = list(new_allowed_datastores) if ds_to_add or ds_to_remove: results['changed'] = True - results['added_datastores'] = ds_to_add - results['removed_datastores'] = ds_to_remove + results['added_datastores'] = list(ds_to_add) + results['removed_datastores'] = list(ds_to_remove) if not module.check_mode: results['reconfig_task_result'] = vmware_cluster_vcls.configure_vcls(ds_to_add, ds_to_remove) diff --git a/tests/integration/targets/init.sh b/tests/integration/targets/init.sh index 68653756..718b0ad3 100644 --- a/tests/integration/targets/init.sh +++ b/tests/integration/targets/init.sh @@ -3,7 +3,6 @@ export DEFAULT_COLLECTIONS_PATH="$ANSIBLE_COLLECTIONS_PATH/ansible_collections" -export ANSIBLE_TURBO_LOOKUP_TTL=1 export ENABLE_TURBO_MODE=1 From 096eed0259cb98f6f7d9eaa4ae6f6d3250957498 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Wed, 30 Oct 2024 11:31:20 -0400 Subject: [PATCH 07/17] test --- .../module_utils/_vmware_ansible_module.py | 30 ++++++++++++++++--- plugins/module_utils/_vmware_facts.py | 14 ++++----- plugins/module_utils/_vmware_tasks.py | 3 +- plugins/modules/cluster_vcls.py | 6 ++-- plugins/modules/guest_info.py | 2 +- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 0c231dda..ee8041d0 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -6,6 +6,7 @@ import os from ansible.module_utils.common.validation import check_type_bool +from ansible.module_utils.common.text.converters import to_native try: enable_turbo_mode = check_type_bool(os.environ.get("ENABLE_TURBO_MODE")) @@ -15,11 +16,32 @@ if enable_turbo_mode: try: from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 - AnsibleTurboModule as AnsibleModule, + AnsibleTurboModule as BaseAnsibleModule, ) - AnsibleModule.collection_name = "vmware.vmware" + BaseAnsibleModule.collection_name = "vmware.vmware" except ImportError: - from ansible.module_utils.basic import AnsibleModule # noqa: F401 + from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 else: - from ansible.module_utils.basic import AnsibleModule # noqa: F401 + from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 + + +class AnsibleModule(BaseAnsibleModule): + def exit_json(self, **kwargs): + if enable_turbo_mode: + kwargs = self.__format_value_for_turbo_server(kwargs) + super().exit_json(**kwargs) + + def __format_value_for_turbo_server(self, value): + if isinstance(value, (str, bool, int)): + return value + if isinstance(value, set): + return self.__format_value_for_turbo_server(list(value)) + if isinstance(value, list): + return [self.__format_value_for_turbo_server(v) for v in value] + if isinstance(value, dict): + for k, v in value.items(): + value[k] = self.__format_value_for_turbo_server(v) + return value + + return to_native(value) diff --git a/plugins/module_utils/_vmware_facts.py b/plugins/module_utils/_vmware_facts.py index 2ae2d071..642b5401 100644 --- a/plugins/module_utils/_vmware_facts.py +++ b/plugins/module_utils/_vmware_facts.py @@ -19,9 +19,9 @@ except ImportError: pass -from ansible.module_utils._text import to_text +from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.six import integer_types, string_types, iteritems -import ansible.module_utils.common._collections_compat as collections_compat +import ansible.module_utils.six.moves.collections_abc as collections_compat from ansible_collections.vmware.vmware.plugins.module_utils._vmware_folder_paths import get_folder_path_of_vm @@ -355,7 +355,7 @@ def serialize_spec(clonespec): elif isinstance(xo, vim.vm.device.VirtualDisk): data[x] = serialize_spec(xo) elif isinstance(xo, vim.vm.device.VirtualDeviceSpec.FileOperation): - data[x] = to_text(xo) + data[x] = to_native(xo) elif isinstance(xo, vim.Description): data[x] = { 'dynamicProperty': serialize_spec(xo.dynamicProperty), @@ -364,7 +364,7 @@ def serialize_spec(clonespec): 'summary': serialize_spec(xo.summary), } elif hasattr(xo, 'name'): - data[x] = to_text(xo) + ':' + to_text(xo.name) + data[x] = to_native(xo) + ':' + to_native(xo.name) elif isinstance(xo, vim.vm.ProfileSpec): pass elif issubclass(xt, list): @@ -375,13 +375,13 @@ def serialize_spec(clonespec): if issubclass(xt, integer_types): data[x] = int(xo) else: - data[x] = to_text(xo) + data[x] = to_native(xo) elif issubclass(xt, bool): data[x] = xo elif issubclass(xt, dict): - data[to_text(x)] = {} + data[to_native(x)] = {} for k, v in xo.items(): - k = to_text(k) + k = to_native(k) data[x][k] = serialize_spec(v) else: data[x] = str(xt) diff --git a/plugins/module_utils/_vmware_tasks.py b/plugins/module_utils/_vmware_tasks.py index 9853b1f5..a8cc369f 100644 --- a/plugins/module_utils/_vmware_tasks.py +++ b/plugins/module_utils/_vmware_tasks.py @@ -14,7 +14,6 @@ import traceback from random import randint -from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.six import raise_from PYVMOMI_IMP_ERR = None @@ -60,7 +59,7 @@ def wait_for_completion(self, max_backoff=64, timeout=3600, vm=None, answers=Non if self.is_task_finished_with_success(): out = { - 'completion_time': to_native(self.task.info.completeTime), + 'completion_time': self.task.info.completeTime, 'state': self.task.info.state, 'result': self.task.info.result, 'entity_name': self.task.info.entityName, diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index 1c70bd58..b57c16de 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -276,11 +276,11 @@ def main(): vmware_cluster_vcls = VMwareClusterVcls(module) ds_to_add, ds_to_remove, new_allowed_datastores = vmware_cluster_vcls.resolve_datastores_to_add_and_remove() - results['allowed_datastores'] = list(new_allowed_datastores) + results['allowed_datastores'] = new_allowed_datastores if ds_to_add or ds_to_remove: results['changed'] = True - results['added_datastores'] = list(ds_to_add) - results['removed_datastores'] = list(ds_to_remove) + results['added_datastores'] = ds_to_add + results['removed_datastores'] = ds_to_remove if not module.check_mode: results['reconfig_task_result'] = vmware_cluster_vcls.configure_vcls(ds_to_add, ds_to_remove) diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index 34b12759..f9681e7e 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -170,7 +170,7 @@ from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( - AnsibleModule, + AnsibleModule ) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient From 4367dfcc5f9c5638a0caea5f10ff00d62d7878e9 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Wed, 30 Oct 2024 12:19:25 -0400 Subject: [PATCH 08/17] test --- plugins/module_utils/_vmware.py | 14 +++++++++++++- plugins/module_utils/_vmware_rest_client.py | 8 ++++++++ plugins/modules/guest_info.py | 2 +- tests/integration/targets/init.sh | 2 +- tests/integration/targets/runme.sh | 6 ++++++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/plugins/module_utils/_vmware.py b/plugins/module_utils/_vmware.py index 69f57189..37e6cdd8 100644 --- a/plugins/module_utils/_vmware.py +++ b/plugins/module_utils/_vmware.py @@ -33,6 +33,7 @@ HAS_PYVMOMI = False from ansible.module_utils.basic import env_fallback, missing_required_lib +import functools class ApiAccessError(Exception): @@ -80,6 +81,7 @@ def vmware_argument_spec(): ) +@functools.lru_cache def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None, port=None, validate_certs=None, httpProxyHost=None, httpProxyPort=None): @@ -203,11 +205,21 @@ def __init__(self, module): self.module = module self.params = module.params self.current_vm_obj = None - self.si, self.content = connect_to_api(self.module, return_si=True) + self.si, self.content = self._connect_to_vcenter() self.custom_field_mgr = [] if self.content.customFieldsManager: # not an ESXi self.custom_field_mgr = self.content.customFieldsManager.field + def __eq__(self, value): + return True + + def __hash__(self): + return hash(self.params['hostname']) + + @functools.lru_cache + def _connect_to_vcenter(self): + return connect_to_api(self.module, return_si=True) + def is_vcenter(self): """ Check if given hostname is vCenter or ESXi host diff --git a/plugins/module_utils/_vmware_rest_client.py b/plugins/module_utils/_vmware_rest_client.py index 0b700472..2028ff67 100644 --- a/plugins/module_utils/_vmware_rest_client.py +++ b/plugins/module_utils/_vmware_rest_client.py @@ -52,6 +52,7 @@ from ansible.module_utils.basic import env_fallback, missing_required_lib from ansible.module_utils._text import to_native +import functools class VmwareRestClient(object): def __init__(self, module): @@ -125,6 +126,13 @@ def vmware_client_argument_spec(): fallback=(env_fallback, ['VMWARE_PROXY_PORT'])), ) + def __eq__(self, value): + return True + + def __hash__(self): + return hash(self.params['hostname']) + + @functools.lru_cache def connect_to_vsphere_client(self): """ Connect to vSphere API Client with Username and Password diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index f9681e7e..5b2a1a10 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -241,7 +241,7 @@ def gather_info_for_guests(self): guest_info['env'] = self._get_env(guest) all_guest_info += [guest_info] - + #all_guest_info = [{'one': self.connect_to_vsphere_client.cache_info(), 'two': self.pyvmomi._connect_to_vcenter.cache_info()}] return all_guest_info def get_guests(self): diff --git a/tests/integration/targets/init.sh b/tests/integration/targets/init.sh index 718b0ad3..6be0ffd2 100644 --- a/tests/integration/targets/init.sh +++ b/tests/integration/targets/init.sh @@ -4,7 +4,7 @@ export DEFAULT_COLLECTIONS_PATH="$ANSIBLE_COLLECTIONS_PATH/ansible_collections" export ENABLE_TURBO_MODE=1 - +export ANSIBLE_TURBO_LOOKUP_TTL=600 # Check if the variable is already set (e.g., in CI) if [ -z "$ANSIBLE_COLLECTIONS_PATH" ]; then diff --git a/tests/integration/targets/runme.sh b/tests/integration/targets/runme.sh index ae85ee92..16193916 100755 --- a/tests/integration/targets/runme.sh +++ b/tests/integration/targets/runme.sh @@ -15,3 +15,9 @@ else echo "ANSIBLE_TAGS is not set for Eco vCenter. Running on simulator." exec ansible-playbook run.yml --tags integration-ci fi +lastExitCode=$? + +# kill turbo server +ps -ef | grep turbo | grep -v 'grep' |awk '{print $2}' | xargs kill -9 + +exit $? From c46fdefb3a1d63d8e64bddd3adec236f0fa908a4 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Wed, 30 Oct 2024 12:26:12 -0400 Subject: [PATCH 09/17] use same turbo server throughout tests --- .github/workflows/eco-vcenter-ci.yaml | 6 ++++++ tests/integration/targets/runme.sh | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eco-vcenter-ci.yaml b/.github/workflows/eco-vcenter-ci.yaml index 119d483e..c664f8e7 100644 --- a/.github/workflows/eco-vcenter-ci.yaml +++ b/.github/workflows/eco-vcenter-ci.yaml @@ -44,3 +44,9 @@ jobs: working-directory: ansible_collections/vmware/vmware env: ANSIBLE_COLLECTIONS_PATH: "${{ github.workspace }}" + + - name: kill turbo server + if: ${{ always() }} + run: | + # kill turbo server + ps -ef | grep turbo | grep -v 'grep' |awk '{print $2}' | xargs kill -9 diff --git a/tests/integration/targets/runme.sh b/tests/integration/targets/runme.sh index 16193916..ae85ee92 100755 --- a/tests/integration/targets/runme.sh +++ b/tests/integration/targets/runme.sh @@ -15,9 +15,3 @@ else echo "ANSIBLE_TAGS is not set for Eco vCenter. Running on simulator." exec ansible-playbook run.yml --tags integration-ci fi -lastExitCode=$? - -# kill turbo server -ps -ef | grep turbo | grep -v 'grep' |awk '{print $2}' | xargs kill -9 - -exit $? From 1a2ce84c62165bf5ec8502f0be89204305479204 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 08:06:27 -0500 Subject: [PATCH 10/17] test --- docs/module_profiling.md | 46 +++++++++++++ docs/turbo_server.md | 69 +++++++++++++++++++ plugins/module_utils/_vmware.py | 14 ++-- .../module_utils/_vmware_ansible_module.py | 17 +++-- plugins/module_utils/_vmware_facts.py | 8 +++ plugins/module_utils/_vmware_rest_client.py | 14 ++-- plugins/modules/guest_info.py | 1 + tests/integration/targets/init.sh | 2 +- 8 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 docs/module_profiling.md create mode 100644 docs/turbo_server.md diff --git a/docs/module_profiling.md b/docs/module_profiling.md new file mode 100644 index 00000000..ca8f6f80 --- /dev/null +++ b/docs/module_profiling.md @@ -0,0 +1,46 @@ +# Module Profiling and Performance + +It might be useful to check module performance when developing. + +## Timing Tasks + +Ansible comes with a basic profiler that outputs how long each task takes. You can enable it by setting an environment variable: +```bash +export ANSIBLE_CALLBACKS_ENABLED=profile_tasks +``` + +## Profiling Modules + +Python comes with a profiler that will show how long each method takes. Heres a very basic example: +```python +def main(): + # import the profiler and setup the context manager + import cProfile + import pstats + with cProfile.Profile() as pr: + argument_spec = VmwareRestClient.vmware_client_argument_spec() + module = AnsibleModule( + argument_spec=argument_spec, + supports_check_mode=True, + ) + + # Execute your module as usual + my_class = Foo(module) + out = my_class.do() + + # save the profile results to a variable + stats = pstats.Stats(pr) + + # sort by time and write the output to a file so you can view the results + stats.sort_stats(pstats.SortKey.TIME) + stats.dump_stats("/some/local/path/profile.prof") + + # exit module as usual + module.exit_json(changed=False, out=out) +``` + +You can use a tool like `snakeviz` to view the profile results. +```bash +pip install snakeviz +snakviz /some/local/path/profile.prof +``` diff --git a/docs/turbo_server.md b/docs/turbo_server.md new file mode 100644 index 00000000..25913dd9 --- /dev/null +++ b/docs/turbo_server.md @@ -0,0 +1,69 @@ +# Using the Turbo Server and Cache + +The VMware modules make a lot of API calls that can take a (relatively) long time to finish. To help speed things up, you can enable some caching functionality. + +The cache relies on the turbo server provided by `cloud.common`. (ADD LINK HERE) + +## Enabling and Configuring + +To enable the cache, set the environment variable: +```bash +export ENABLE_TURBO_SERVER=1 +``` + +The cache expires every 15 seconds by default. To change the length of time before the cache expires, set the environment variable: +```bash +export ANSIBLE_TURBO_LOOKUP_TTL=120 +``` + +You can also set these variables in your playbook, if that's more convenient: +```yaml +- name: Example + hosts: localhost + environment: + ENABLE_TURBO_SERVER: 1 + ANSIBLE_TURBO_LOOKUP_TTL: 120 + tasks: + ... +``` + +## Development + +To use the turbo server in your module, you need to replace the AnsibleModule import with the custom class from this repo. +```python +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule +) +``` + +You can leverage the cache from `functools` to save the results from a method. One use case would be caching an authentication session or the results from looking up VM information. To use the cache, import `functools` and then add the cache decorator to the method: +```python +import functools + +@functools.cache +def my_method(): + .... +``` + +When attaching the cache decorator to methods, the argument inputs are hashed and compared to previous calls to determine if a cached result can be used. This means if your method uses a class, the class must have a reasonable hash and equals method defined. For example: +```python +class PyVmomi(object): + def __init__(self, module): + """ + Constructor + """ + self.module = module + self.params = module.params + .... + + def __eq__(self, value): + if not isinstance(value, self.__class__): + return False + return bool(all([ + (self.params['hostname'] == value.params['hostname']), + (self.params['username'] == value.params['username']) + ])) + + def __hash__(self): + return hash(self.params['hostname'] + self.params['username']) +``` diff --git a/plugins/module_utils/_vmware.py b/plugins/module_utils/_vmware.py index 37e6cdd8..e17a9589 100644 --- a/plugins/module_utils/_vmware.py +++ b/plugins/module_utils/_vmware.py @@ -81,7 +81,7 @@ def vmware_argument_spec(): ) -@functools.lru_cache +@functools.cache def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None, port=None, validate_certs=None, httpProxyHost=None, httpProxyPort=None): @@ -211,12 +211,17 @@ def __init__(self, module): self.custom_field_mgr = self.content.customFieldsManager.field def __eq__(self, value): - return True + if not isinstance(value, self.__class__): + return False + return bool(all([ + (self.params['hostname'] == value.params['hostname']), + (self.params['username'] == value.params['username']) + ])) def __hash__(self): - return hash(self.params['hostname']) + return hash(self.params['hostname'] + self.params['username']) - @functools.lru_cache + @functools.cache def _connect_to_vcenter(self): return connect_to_api(self.module, return_si=True) @@ -294,6 +299,7 @@ def get_dvs_portgroup(self, portgroup): """ return self.get_objs_by_name_or_moid([vim.dvs.DistributedVirtualPortgroup], portgroup) + @functools.cache def get_vm_using_params( self, name_param='name', uuid_param='uuid', moid_param='moid', fail_on_missing=False, name_match_param='name_match', use_instance_uuid_param='use_instance_uuid'): diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index ee8041d0..8a5aeb59 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -8,10 +8,8 @@ from ansible.module_utils.common.validation import check_type_bool from ansible.module_utils.common.text.converters import to_native -try: - enable_turbo_mode = check_type_bool(os.environ.get("ENABLE_TURBO_MODE")) -except TypeError: - enable_turbo_mode = False + +enable_turbo_mode = check_type_bool(os.environ.get("ENABLE_TURBO_MODE", False)) if enable_turbo_mode: try: @@ -21,12 +19,19 @@ BaseAnsibleModule.collection_name = "vmware.vmware" except ImportError: - from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 + from ansible.module_utils.basic import BaseAnsibleModule else: - from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 + from ansible.module_utils.basic import BaseAnsibleModule class AnsibleModule(BaseAnsibleModule): + """ + This should really be added to the upstream cloud.common repo, but until then we need it here. + The outputs from a module need to be passed through the turbo server using pickle. If the output + contains something that pickle cannot encode/decode, we need to convert it first. + For most APIs that return content as JSON, this isn't an issue. But for the SDKs VMware uses, + it can be a problem. + """ def exit_json(self, **kwargs): if enable_turbo_mode: kwargs = self.__format_value_for_turbo_server(kwargs) diff --git a/plugins/module_utils/_vmware_facts.py b/plugins/module_utils/_vmware_facts.py index 642b5401..aac05cb7 100644 --- a/plugins/module_utils/_vmware_facts.py +++ b/plugins/module_utils/_vmware_facts.py @@ -12,6 +12,7 @@ import json import os +import functools PYVMOMI_IMP_ERR = None try: @@ -29,6 +30,12 @@ class VmFacts(): def __init__(self, vm): self.vm = vm + def __eq__(self, value): + return True + + def __hash__(self): + return hash(self.vm._GetMoId()) + def hw_all_facts(self): ''' Returns a combined set of all 'hw_' facts @@ -42,6 +49,7 @@ def hw_all_facts(self): **self.hw_network_device_facts() } + @functools.cache def all_facts(self, content): return { **self.hw_all_facts(), diff --git a/plugins/module_utils/_vmware_rest_client.py b/plugins/module_utils/_vmware_rest_client.py index 2028ff67..70ee50cc 100644 --- a/plugins/module_utils/_vmware_rest_client.py +++ b/plugins/module_utils/_vmware_rest_client.py @@ -12,6 +12,7 @@ __metaclass__ = type import traceback +import functools REQUESTS_IMP_ERR = None try: @@ -52,7 +53,6 @@ from ansible.module_utils.basic import env_fallback, missing_required_lib from ansible.module_utils._text import to_native -import functools class VmwareRestClient(object): def __init__(self, module): @@ -127,12 +127,17 @@ def vmware_client_argument_spec(): ) def __eq__(self, value): - return True + if not isinstance(value, self.__class__): + return False + return bool(all([ + (self.params['hostname'] == value.params['hostname']), + (self.params['username'] == value.params['username']) + ])) def __hash__(self): - return hash(self.params['hostname']) + return hash(self.params['hostname'] + self.params['username']) - @functools.lru_cache + @functools.cache def connect_to_vsphere_client(self): """ Connect to vSphere API Client with Username and Password @@ -179,6 +184,7 @@ def connect_to_vsphere_client(self): return client + @functools.cache def get_vm_by_name(self, name): """ Returns a VM object that matches the given name. diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index 5b2a1a10..248689c4 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -304,6 +304,7 @@ def main(): vmware_appliance_mgr = VmwareGuestInfo(module) guests = vmware_appliance_mgr.gather_info_for_guests() + module.exit_json(changed=False, guests=guests) diff --git a/tests/integration/targets/init.sh b/tests/integration/targets/init.sh index 6be0ffd2..6852054e 100644 --- a/tests/integration/targets/init.sh +++ b/tests/integration/targets/init.sh @@ -4,7 +4,7 @@ export DEFAULT_COLLECTIONS_PATH="$ANSIBLE_COLLECTIONS_PATH/ansible_collections" export ENABLE_TURBO_MODE=1 -export ANSIBLE_TURBO_LOOKUP_TTL=600 +export ANSIBLE_TURBO_LOOKUP_TTL=120 # Check if the variable is already set (e.g., in CI) if [ -z "$ANSIBLE_COLLECTIONS_PATH" ]; then From 1468928974dda1a8861f07778ffec05127ef4f3d Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 08:15:29 -0500 Subject: [PATCH 11/17] test fixes --- galaxy.yml | 2 ++ plugins/module_utils/_vmware.py | 6 +++--- plugins/module_utils/_vmware_ansible_module.py | 4 ++-- plugins/module_utils/_vmware_facts.py | 2 +- plugins/module_utils/_vmware_rest_client.py | 4 ++-- plugins/module_utils/_vmware_tasks.py | 1 + 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index f558d557..16750d90 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -22,3 +22,5 @@ build_ignore: - changelogs/.plugin-cache.yaml - .github - .vscode +dependencies: + cloud.common: "*" diff --git a/plugins/module_utils/_vmware.py b/plugins/module_utils/_vmware.py index e17a9589..67fe6f0f 100644 --- a/plugins/module_utils/_vmware.py +++ b/plugins/module_utils/_vmware.py @@ -81,7 +81,7 @@ def vmware_argument_spec(): ) -@functools.cache +@functools.cache(maxsize=128) def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None, port=None, validate_certs=None, httpProxyHost=None, httpProxyPort=None): @@ -221,7 +221,7 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.cache + @functools.cache(maxsize=128) def _connect_to_vcenter(self): return connect_to_api(self.module, return_si=True) @@ -299,7 +299,7 @@ def get_dvs_portgroup(self, portgroup): """ return self.get_objs_by_name_or_moid([vim.dvs.DistributedVirtualPortgroup], portgroup) - @functools.cache + @functools.cache(maxsize=128) def get_vm_using_params( self, name_param='name', uuid_param='uuid', moid_param='moid', fail_on_missing=False, name_match_param='name_match', use_instance_uuid_param='use_instance_uuid'): diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 8a5aeb59..ef390696 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -19,9 +19,9 @@ BaseAnsibleModule.collection_name = "vmware.vmware" except ImportError: - from ansible.module_utils.basic import BaseAnsibleModule + from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 else: - from ansible.module_utils.basic import BaseAnsibleModule + from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 class AnsibleModule(BaseAnsibleModule): diff --git a/plugins/module_utils/_vmware_facts.py b/plugins/module_utils/_vmware_facts.py index aac05cb7..97122219 100644 --- a/plugins/module_utils/_vmware_facts.py +++ b/plugins/module_utils/_vmware_facts.py @@ -49,7 +49,7 @@ def hw_all_facts(self): **self.hw_network_device_facts() } - @functools.cache + @functools.cache(maxsize=128) def all_facts(self, content): return { **self.hw_all_facts(), diff --git a/plugins/module_utils/_vmware_rest_client.py b/plugins/module_utils/_vmware_rest_client.py index 70ee50cc..e2c16b10 100644 --- a/plugins/module_utils/_vmware_rest_client.py +++ b/plugins/module_utils/_vmware_rest_client.py @@ -137,7 +137,7 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.cache + @functools.cache(maxsize=128) def connect_to_vsphere_client(self): """ Connect to vSphere API Client with Username and Password @@ -184,7 +184,7 @@ def connect_to_vsphere_client(self): return client - @functools.cache + @functools.cache(maxsize=128) def get_vm_by_name(self, name): """ Returns a VM object that matches the given name. diff --git a/plugins/module_utils/_vmware_tasks.py b/plugins/module_utils/_vmware_tasks.py index a8cc369f..29d1a75f 100644 --- a/plugins/module_utils/_vmware_tasks.py +++ b/plugins/module_utils/_vmware_tasks.py @@ -15,6 +15,7 @@ from random import randint from ansible.module_utils.six import raise_from +from ansible.module_utils.common.text.converters import to_native PYVMOMI_IMP_ERR = None try: From d12ca4a982ac4cf4567df330181638ace4e5ce97 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 08:33:53 -0500 Subject: [PATCH 12/17] sanity fixes --- plugins/module_utils/_vmware.py | 8 ++++---- plugins/module_utils/_vmware_ansible_module.py | 6 +++--- plugins/module_utils/_vmware_facts.py | 2 +- plugins/module_utils/_vmware_rest_client.py | 4 ++-- plugins/modules/cluster_vcls.py | 1 - plugins/modules/guest_info.py | 1 - 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/plugins/module_utils/_vmware.py b/plugins/module_utils/_vmware.py index 67fe6f0f..e26838b3 100644 --- a/plugins/module_utils/_vmware.py +++ b/plugins/module_utils/_vmware.py @@ -81,7 +81,7 @@ def vmware_argument_spec(): ) -@functools.cache(maxsize=128) +@functools.lru_cache(maxsize=128) def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None, port=None, validate_certs=None, httpProxyHost=None, httpProxyPort=None): @@ -221,9 +221,9 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.cache(maxsize=128) + @functools.lru_cache(maxsize=128) def _connect_to_vcenter(self): - return connect_to_api(self.module, return_si=True) + return connect_to_api(self.module, return_si=True) def is_vcenter(self): """ @@ -299,7 +299,7 @@ def get_dvs_portgroup(self, portgroup): """ return self.get_objs_by_name_or_moid([vim.dvs.DistributedVirtualPortgroup], portgroup) - @functools.cache(maxsize=128) + @functools.lru_cache(maxsize=128) def get_vm_using_params( self, name_param='name', uuid_param='uuid', moid_param='moid', fail_on_missing=False, name_match_param='name_match', use_instance_uuid_param='use_instance_uuid'): diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index ef390696..ac00f842 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -13,15 +13,15 @@ if enable_turbo_mode: try: - from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( # noqa: F401 + from ansible_collections.cloud.common.plugins.module_utils.turbo.module import ( AnsibleTurboModule as BaseAnsibleModule, ) BaseAnsibleModule.collection_name = "vmware.vmware" except ImportError: - from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 + from ansible.module_utils.basic import AnsibleModule as BaseAnsibleModule else: - from ansible.module_utils.basic import BaseAnsibleModule # noqa: F401 + from ansible.module_utils.basic import AnsibleModule as BaseAnsibleModule class AnsibleModule(BaseAnsibleModule): diff --git a/plugins/module_utils/_vmware_facts.py b/plugins/module_utils/_vmware_facts.py index 97122219..779903ac 100644 --- a/plugins/module_utils/_vmware_facts.py +++ b/plugins/module_utils/_vmware_facts.py @@ -49,7 +49,7 @@ def hw_all_facts(self): **self.hw_network_device_facts() } - @functools.cache(maxsize=128) + @functools.lru_cache(maxsize=128) def all_facts(self, content): return { **self.hw_all_facts(), diff --git a/plugins/module_utils/_vmware_rest_client.py b/plugins/module_utils/_vmware_rest_client.py index e2c16b10..fc6b0df1 100644 --- a/plugins/module_utils/_vmware_rest_client.py +++ b/plugins/module_utils/_vmware_rest_client.py @@ -137,7 +137,7 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.cache(maxsize=128) + @functools.lru_cache(maxsize=128) def connect_to_vsphere_client(self): """ Connect to vSphere API Client with Username and Password @@ -184,7 +184,7 @@ def connect_to_vsphere_client(self): return client - @functools.cache(maxsize=128) + @functools.lru_cache(maxsize=128) def get_vm_by_name(self, name): """ Returns a VM object that matches the given name. diff --git a/plugins/modules/cluster_vcls.py b/plugins/modules/cluster_vcls.py index b57c16de..e2b8b0fe 100644 --- a/plugins/modules/cluster_vcls.py +++ b/plugins/modules/cluster_vcls.py @@ -148,7 +148,6 @@ ) - class VMwareClusterVcls(PyVmomi): def __init__(self, module): super(VMwareClusterVcls, self).__init__(module) diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index 248689c4..8cdaa3bd 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -241,7 +241,6 @@ def gather_info_for_guests(self): guest_info['env'] = self._get_env(guest) all_guest_info += [guest_info] - #all_guest_info = [{'one': self.connect_to_vsphere_client.cache_info(), 'two': self.pyvmomi._connect_to_vcenter.cache_info()}] return all_guest_info def get_guests(self): From ba4692b06d3d7e374b4036600cf641f6c2ed9430 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 11:03:36 -0500 Subject: [PATCH 13/17] test --- plugins/module_utils/_vmware.py | 10 ++-- .../module_utils/_vmware_ansible_module.py | 31 +++++++++- plugins/module_utils/_vmware_facts.py | 12 +++- plugins/module_utils/_vmware_rest_client.py | 8 ++- plugins/modules/clear_cache.py | 58 +++++++++++++++++++ plugins/modules/guest_info.py | 6 +- .../tasks/main.yml | 3 + 7 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 plugins/modules/clear_cache.py diff --git a/plugins/module_utils/_vmware.py b/plugins/module_utils/_vmware.py index e26838b3..e8cc39c4 100644 --- a/plugins/module_utils/_vmware.py +++ b/plugins/module_utils/_vmware.py @@ -33,7 +33,9 @@ HAS_PYVMOMI = False from ansible.module_utils.basic import env_fallback, missing_required_lib -import functools +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + cache, +) class ApiAccessError(Exception): @@ -81,7 +83,7 @@ def vmware_argument_spec(): ) -@functools.lru_cache(maxsize=128) +@cache def connect_to_api(module, disconnect_atexit=True, return_si=False, hostname=None, username=None, password=None, port=None, validate_certs=None, httpProxyHost=None, httpProxyPort=None): @@ -221,7 +223,7 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.lru_cache(maxsize=128) + @cache def _connect_to_vcenter(self): return connect_to_api(self.module, return_si=True) @@ -299,7 +301,7 @@ def get_dvs_portgroup(self, portgroup): """ return self.get_objs_by_name_or_moid([vim.dvs.DistributedVirtualPortgroup], portgroup) - @functools.lru_cache(maxsize=128) + @cache def get_vm_using_params( self, name_param='name', uuid_param='uuid', moid_param='moid', fail_on_missing=False, name_match_param='name_match', use_instance_uuid_param='use_instance_uuid'): diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index ac00f842..213e5642 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -4,6 +4,7 @@ import os +import functools from ansible.module_utils.common.validation import check_type_bool from ansible.module_utils.common.text.converters import to_native @@ -24,9 +25,12 @@ from ansible.module_utils.basic import AnsibleModule as BaseAnsibleModule +CACHED_FUNCTION_REGISTRY = set() + class AnsibleModule(BaseAnsibleModule): """ - This should really be added to the upstream cloud.common repo, but until then we need it here. + The exit_json and __format_value_for_turbo_server should really be added to the upstream + cloud.common repo, but until then we need it here. The outputs from a module need to be passed through the turbo server using pickle. If the output contains something that pickle cannot encode/decode, we need to convert it first. For most APIs that return content as JSON, this isn't an issue. But for the SDKs VMware uses, @@ -50,3 +54,28 @@ def __format_value_for_turbo_server(self, value): return value return to_native(value) + + def clear_vmware_cache(self, funcs=None): + cleared = set() + no_cache = set() + if not funcs: + funcs = CACHED_FUNCTION_REGISTRY + + for f in funcs: + try: + f.cache_clear() + cleared.add(f.__name__) + except AttributeError: + no_cache.add(f.__name__) + + return cleared, no_cache + + +def cache(func): + @functools.wraps(func) + @functools.lru_cache(maxsize=128) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + CACHED_FUNCTION_REGISTRY.add(wrapper) + return wrapper diff --git a/plugins/module_utils/_vmware_facts.py b/plugins/module_utils/_vmware_facts.py index 779903ac..03b6398d 100644 --- a/plugins/module_utils/_vmware_facts.py +++ b/plugins/module_utils/_vmware_facts.py @@ -12,7 +12,6 @@ import json import os -import functools PYVMOMI_IMP_ERR = None try: @@ -24,6 +23,9 @@ from ansible.module_utils.six import integer_types, string_types, iteritems import ansible.module_utils.six.moves.collections_abc as collections_compat from ansible_collections.vmware.vmware.plugins.module_utils._vmware_folder_paths import get_folder_path_of_vm +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + cache, +) class VmFacts(): @@ -31,7 +33,11 @@ def __init__(self, vm): self.vm = vm def __eq__(self, value): - return True + if not isinstance(value, self.__class__): + return False + return bool(all([ + (self.vm._GetMoId() == value.vm._GetMoId()) + ])) def __hash__(self): return hash(self.vm._GetMoId()) @@ -49,7 +55,7 @@ def hw_all_facts(self): **self.hw_network_device_facts() } - @functools.lru_cache(maxsize=128) + @cache def all_facts(self, content): return { **self.hw_all_facts(), diff --git a/plugins/module_utils/_vmware_rest_client.py b/plugins/module_utils/_vmware_rest_client.py index fc6b0df1..b235a9ae 100644 --- a/plugins/module_utils/_vmware_rest_client.py +++ b/plugins/module_utils/_vmware_rest_client.py @@ -12,7 +12,6 @@ __metaclass__ = type import traceback -import functools REQUESTS_IMP_ERR = None try: @@ -52,6 +51,9 @@ from ansible.module_utils.basic import env_fallback, missing_required_lib from ansible.module_utils._text import to_native +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + cache, +) class VmwareRestClient(object): @@ -137,7 +139,7 @@ def __eq__(self, value): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) - @functools.lru_cache(maxsize=128) + @cache def connect_to_vsphere_client(self): """ Connect to vSphere API Client with Username and Password @@ -184,7 +186,7 @@ def connect_to_vsphere_client(self): return client - @functools.lru_cache(maxsize=128) + @cache def get_vm_by_name(self, name): """ Returns a VM object that matches the given name. diff --git a/plugins/modules/clear_cache.py b/plugins/modules/clear_cache.py new file mode 100644 index 00000000..8c57d3a8 --- /dev/null +++ b/plugins/modules/clear_cache.py @@ -0,0 +1,58 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2023, Ansible Cloud Team (@ansible-collections) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: clear_cache +short_description: Clears the internal cache used by Ansible vmware.vmware +description: +- This module clears the internal cache used by Ansible vmware.vmware modules, if caching is enabled. +author: +- Ansible Cloud Team (@ansible-collections) +requirements: [] +options: {} + +attributes: + check_mode: + description: The check_mode support. + support: none +extends_documentation_fragment: [] +''' + +EXAMPLES = r''' +- name: Clear the cache + vmware.vmware.clear_cache: {} +''' + +RETURN = r''' +''' + + +from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( + AnsibleModule, +) + + +def main(): + module = AnsibleModule( + argument_spec={}, + supports_check_mode=True, + ) + + cleared, no_cache = module.clear_vmware_cache() + module.exit_json( + changed=bool(cleared), + cleared=list(cleared), + no_cache=list(no_cache) + ) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index 8cdaa3bd..fcb70c41 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -170,7 +170,7 @@ from ansible_collections.vmware.vmware.plugins.module_utils._vmware_ansible_module import ( - AnsibleModule + AnsibleModule, ) from ansible_collections.vmware.vmware.plugins.module_utils._vmware import PyVmomi from ansible_collections.vmware.vmware.plugins.module_utils._vmware_rest_client import VmwareRestClient @@ -302,8 +302,8 @@ def main(): module.fail_json(msg="The option 'properties' is only valid when the schema is 'vsphere'") vmware_appliance_mgr = VmwareGuestInfo(module) - guests = vmware_appliance_mgr.gather_info_for_guests() - + #guests = vmware_appliance_mgr.gather_info_for_guests() + guests = [] module.exit_json(changed=False, guests=guests) diff --git a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml index 3050a4c3..120bdd68 100644 --- a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml +++ b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml @@ -120,6 +120,9 @@ state: absent loop: "{{ template_folders }}" + - name: Clear the cache + vmware.vmware.clear_cache: {} + - name: Verify template deletion ansible.builtin.include_tasks: verify_template_deletion.yml loop: "{{ template_folders }}" From 82b0d91bd450e8093cfebf15a3cd198ad400a946 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 11:12:33 -0500 Subject: [PATCH 14/17] clear the cache more --- plugins/modules/guest_info.py | 3 +-- .../targets/vmware_folder_template_from_vm/tasks/main.yml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/modules/guest_info.py b/plugins/modules/guest_info.py index fcb70c41..66b20d47 100644 --- a/plugins/modules/guest_info.py +++ b/plugins/modules/guest_info.py @@ -302,8 +302,7 @@ def main(): module.fail_json(msg="The option 'properties' is only valid when the schema is 'vsphere'") vmware_appliance_mgr = VmwareGuestInfo(module) - #guests = vmware_appliance_mgr.gather_info_for_guests() - guests = [] + guests = vmware_appliance_mgr.gather_info_for_guests() module.exit_json(changed=False, guests=guests) diff --git a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml index 120bdd68..7fc75fed 100644 --- a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml +++ b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml @@ -102,6 +102,9 @@ loop: "{{ template_folders }}" register: __res + - name: Clear the cache + vmware.vmware.clear_cache: {} + - name: Verify template creation ansible.builtin.include_tasks: verify_template_creation.yml loop: "{{ template_folders }}" From f25d1ed50c9028250054f29964add81289c31413 Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 11:30:21 -0500 Subject: [PATCH 15/17] sanity fixes --- plugins/module_utils/_vmware_ansible_module.py | 1 + plugins/modules/clear_cache.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/module_utils/_vmware_ansible_module.py b/plugins/module_utils/_vmware_ansible_module.py index 213e5642..f861d175 100644 --- a/plugins/module_utils/_vmware_ansible_module.py +++ b/plugins/module_utils/_vmware_ansible_module.py @@ -27,6 +27,7 @@ CACHED_FUNCTION_REGISTRY = set() + class AnsibleModule(BaseAnsibleModule): """ The exit_json and __format_value_for_turbo_server should really be added to the upstream diff --git a/plugins/modules/clear_cache.py b/plugins/modules/clear_cache.py index 8c57d3a8..29deccf1 100644 --- a/plugins/modules/clear_cache.py +++ b/plugins/modules/clear_cache.py @@ -43,7 +43,7 @@ def main(): module = AnsibleModule( argument_spec={}, - supports_check_mode=True, + supports_check_mode=False, ) cleared, no_cache = module.clear_vmware_cache() From 8ae19a3a7d962ca2f0a4f7c865f74d2364fa1f5e Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 12:07:06 -0500 Subject: [PATCH 16/17] test --- docs/turbo_server.md | 26 +++++++++++++++++++ plugins/modules/folder_template_from_vm.py | 3 +++ .../tasks/main.yml | 8 +++--- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/turbo_server.md b/docs/turbo_server.md index 25913dd9..df12463f 100644 --- a/docs/turbo_server.md +++ b/docs/turbo_server.md @@ -27,6 +27,21 @@ You can also set these variables in your playbook, if that's more convenient: ... ``` +### Clearing The Cache + +You may find the need to clear the cache manually. This will make sure that all cached method return values are invalidated. You can do so with the `clear_cache` module: +```yaml +- name: Clear the cache + vmware.vmware.clear_cache: {} +``` + +### Killing the turbo server + +You may want to kill the turbo server before its expriation time. This will clear the cache and also delete any cached module files. You can do so by terminating the process running on the remote host (the host that the vmware.vmware task was run on): +```bash +ps -ef | grep turbo | grep -v grep | awk '{print $2}' | xargs kill +``` + ## Development To use the turbo server in your module, you need to replace the AnsibleModule import with the custom class from this repo. @@ -67,3 +82,14 @@ class PyVmomi(object): def __hash__(self): return hash(self.params['hostname'] + self.params['username']) ``` + +To clear the cache from within a module, you can call the builtin clear_cache method. If the user is not using the turbo server, the module does nothing so you can call this method safely without checking. +```python +def main(): + module = AnsibleModule( + argument_spec=argument_spec, + supports_check_mode=True, + ) + .... + module.clear_cache() +``` diff --git a/plugins/modules/folder_template_from_vm.py b/plugins/modules/folder_template_from_vm.py index daf322d9..ebda5e75 100644 --- a/plugins/modules/folder_template_from_vm.py +++ b/plugins/modules/folder_template_from_vm.py @@ -300,6 +300,9 @@ def main(): template.Destroy_Task() result['changed'] = True + if result['changed']: + module.clear_cache() + module.exit_json(**result) diff --git a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml index 7fc75fed..a6ed6717 100644 --- a/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml +++ b/tests/integration/targets/vmware_folder_template_from_vm/tasks/main.yml @@ -102,8 +102,8 @@ loop: "{{ template_folders }}" register: __res - - name: Clear the cache - vmware.vmware.clear_cache: {} + # - name: Clear the cache + # vmware.vmware.clear_cache: {} - name: Verify template creation ansible.builtin.include_tasks: verify_template_creation.yml @@ -123,8 +123,8 @@ state: absent loop: "{{ template_folders }}" - - name: Clear the cache - vmware.vmware.clear_cache: {} + # - name: Clear the cache + # vmware.vmware.clear_cache: {} - name: Verify template deletion ansible.builtin.include_tasks: verify_template_deletion.yml From 81f441c356c84e4fe2443abee558e21800ee7e9e Mon Sep 17 00:00:00 2001 From: Mike Morency Date: Mon, 4 Nov 2024 12:28:16 -0500 Subject: [PATCH 17/17] fix module --- docs/turbo_server.md | 4 ++-- plugins/modules/folder_template_from_vm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/turbo_server.md b/docs/turbo_server.md index df12463f..bbb326a5 100644 --- a/docs/turbo_server.md +++ b/docs/turbo_server.md @@ -83,7 +83,7 @@ class PyVmomi(object): return hash(self.params['hostname'] + self.params['username']) ``` -To clear the cache from within a module, you can call the builtin clear_cache method. If the user is not using the turbo server, the module does nothing so you can call this method safely without checking. +To clear the cache from within a module, you can call the builtin `clear_vmware_cache` method. If the user is not using the turbo server, the module does nothing so you can call this method safely without checking. ```python def main(): module = AnsibleModule( @@ -91,5 +91,5 @@ def main(): supports_check_mode=True, ) .... - module.clear_cache() + module.clear_vmware_cache() ``` diff --git a/plugins/modules/folder_template_from_vm.py b/plugins/modules/folder_template_from_vm.py index ebda5e75..73d146c2 100644 --- a/plugins/modules/folder_template_from_vm.py +++ b/plugins/modules/folder_template_from_vm.py @@ -301,7 +301,7 @@ def main(): result['changed'] = True if result['changed']: - module.clear_cache() + module.clear_vmware_cache() module.exit_json(**result)