From 1faee859c47226101c40c3b3d6bf5f8d849ded60 Mon Sep 17 00:00:00 2001 From: EarthmanT Date: Mon, 10 Apr 2023 08:36:19 -0400 Subject: [PATCH] add __version__ file (#239) (#240) * add __version__ file (#239) * add __version__ file * update orbs * support backend migration (#237) * support backend migration * heal diff * treat thing * bump version * Adding notifications. * try to run this test again * Remove unnecessary context. * Remove unnecessary post-step. * add sleep between plan storage and run * attach at workspace * fix more test stuff * fix more test stuff * more test playing * check if debug is where it all started * change test order? * change notifications. * bump examples submodule * log to info diff in plan * update how we calculate diff in tf plugin * improve plan diff * more test fix * Updated config.yml --------- Co-authored-by: Nely Nehemia <39059747+Nelynehemia@users.noreply.github.com> Co-authored-by: Bartosz Kosciug Co-authored-by: bartoszkosciug <97445164+bartoszkosciug@users.noreply.github.com> --- .circleci/config.yml | 21 ++++- .circleci/test_features.py | 105 +++++++++++++++--------- CHANGELOG.txt | 3 + cloudify_tf/__version__.py | 1 + cloudify_tf/tasks.py | 32 ++++++-- cloudify_tf/terraform/__init__.py | 30 +++++-- cloudify_tf/tests/test_plugin.py | 59 ++++++++++++++ cloudify_tf/tests/test_workflows.py | 60 +++++++++++++- cloudify_tf/utils.py | 5 +- cloudify_tf/workflows.py | 33 +++++++- examples/blueprint-examples | 2 +- ignore_plugin_yaml_differences | 2 +- plugin.yaml | 52 +++++++++++- plugin_1_4.yaml | 119 ++++++++++++++++++++-------- setup.py | 26 +++--- test-requirements.txt | 1 - v2_plugin.yaml | 110 +++++++++++++++++-------- 17 files changed, 523 insertions(+), 138 deletions(-) create mode 100644 cloudify_tf/__version__.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 44de739..c897e5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,24 @@ version: 2.1 +job-post-steps: &job-post-steps + post-steps: + - slack/notify_failed + +unittest-post-steps: &unittest-post-steps + post-steps: + - store_test_results: + path: /home/circleci/project/nosetests.xml + - store_artifacts: + path: /home/circleci/project/coverage.xml + prefix: tests + - slack/notify_failed + orbs: node: cloudify/public-unittest-orb@1 #orb version wagonorb: cloudify/wagon-bulder-orb@2 #orb version releaseorb: cloudify/release-orb@1 #orb version managerorb: cloudify/manager-orb@2 + slack: cloudify/notify-slack@2 checkout: post: @@ -31,6 +45,8 @@ commands: prepare_test_manager: steps: + - attach_workspace: + at: workspace - run: ecosystem-test prepare-test-manager -l $TEST_LICENSE --generate-new-aws-token -es aws_access_key_id=$aws_access_key_id -es aws_secret_access_key=$aws_secret_access_key --yum-package python-netaddr --yum-package git -p $(find ~/project/workspace/build/ -name *centos-Core*x86_64.wgn) ~/project/plugin.yaml - run: ecosystem-test upload-plugin -PN utilities @@ -80,8 +96,6 @@ workflows: - integration_tests_py3: requires: - wagonorb/wagon - - wagonorb/arch64_wagon - - wagonorb/rhel_wagon filters: branches: only: /([0-9\.]*\-build|master|dev)/ @@ -127,6 +141,9 @@ workflows: branches: only: /([0-9\.]*\-build|master|dev)/ - integration_tests_py3: + context: + - slack-secrets + <<: *job-post-steps requires: - wagonorb/wagon - wagonorb/rhel_wagon diff --git a/.circleci/test_features.py b/.circleci/test_features.py index 94e139e..0403e61 100644 --- a/.circleci/test_features.py +++ b/.circleci/test_features.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time from os import environ +from json import loads, JSONDecodeError from contextlib import contextmanager import pytest @@ -26,6 +28,24 @@ TEST_ID = environ.get('__ECOSYSTEM_TEST_ID', 'virtual-machine') +source = 'https://github.com/cloudify-community/tf-source/archive/refs/heads/main.zip' # noqa + +public_params = { + 'source': source, + 'source_path': 'template/modules/public_vm', +} + +private_params = { + 'source': source, + 'source_path': 'template/modules/private_vm', +} + +private_params_force = { + 'source': source, + 'source_path': 'template/modules/private_vm', + 'force': False +} + @contextmanager def test_cleaner_upper(): @@ -36,52 +56,29 @@ def test_cleaner_upper(): raise -@pytest.mark.dependency(depends=['test_plan_protection']) -def test_drifts(*_, **__): - with test_cleaner_upper(): - before_props = cloud_resources_node_instance_runtime_properties() - change_a_resource(before_props) - executions_start('refresh_terraform_resources', TEST_ID, 150) - after_props = cloud_resources_node_instance_runtime_properties() - drifts = after_props.get('drifts') - logger.info('Drifts: {drifts}'.format(drifts=drifts)) - if drifts: - return - raise Exception('The test_drifts test failed.') - - @pytest.mark.dependency() def test_plan_protection(*_, **__): with test_cleaner_upper(): - params = { - 'source': 'https://github.com/cloudify-community/tf-source/archive/refs/heads/main.zip', # noqa - 'source_path': 'template/modules/public_vm', - } - executions_start('terraform_plan', TEST_ID, 300, params) + executions_start('terraform_plan', TEST_ID, 300, public_params) logger.info('Wrap plan for public VM. ' 'Now we will run reload_terraform_template for private VM ' 'and it should fail.') - params = { - 'source': 'https://github.com/cloudify-community/tf-source/archive/refs/heads/main.zip', # noqa - 'source_path': 'template/modules/private_vm', - 'force': False - } try: - executions_start('reload_terraform_template', TEST_ID, 300, params) + executions_start( + 'reload_terraform_template', TEST_ID, 300, private_params_force) except EcosystemTestException: - logger.info('Apply caught our plan mismatch.') + logger.info('Apply caught our plan mismatch.'.upper()) else: raise EcosystemTestException( 'Apply did not catch the plan mismatch.') - del params['force'] - executions_start('terraform_plan', TEST_ID, 300, params) - logger.info('Now rerunning apply with a matching plan.') + executions_start('terraform_plan', TEST_ID, 300, private_params) + time.sleep(10) before = cloud_resources_node_instance_runtime_properties() logger.info('Before outputs: {before}'.format( before=before.get('outputs'))) logger.info('Now rerunning plan.') - params['force'] = False - executions_start('reload_terraform_template', TEST_ID, 300, params) + executions_start( + 'reload_terraform_template', TEST_ID, 300, private_params_force) after = cloud_resources_node_instance_runtime_properties() logger.info('After outputs: {after}'.format( after=before.get('outputs'))) @@ -89,12 +86,27 @@ def test_plan_protection(*_, **__): raise Exception('Outputs should not match after reload.') +@pytest.mark.dependency(depends=['test_plan_protection']) +def test_drifts(*_, **__): + with test_cleaner_upper(): + before_props = cloud_resources_node_instance_runtime_properties() + change_a_resource(before_props) + executions_start('refresh_terraform_resources', TEST_ID, 150) + after_props = cloud_resources_node_instance_runtime_properties() + drifts = after_props.get('drifts') + logger.info('Drifts: {drifts}'.format(drifts=drifts)) + if drifts: + return + raise Exception('The test_drifts test failed.') + + def nodes(): return cloudify_exec('cfy nodes list') def node_instances(): - return cloudify_exec('cfy node-instances list -d {}'.format(TEST_ID)) + return cloudify_exec( + 'cfy node-instances list -d {}'.format(TEST_ID), log=False) def node_instance_by_name(name): @@ -106,7 +118,7 @@ def node_instance_by_name(name): def node_instance_runtime_properties(name): node_instance = cloudify_exec( - 'cfy node-instance get {name}'.format(name=name)) + 'cfy node-instance get {name}'.format(name=name), log=False) return node_instance['runtime_properties'] @@ -118,8 +130,6 @@ def cloud_resources_node_instance_runtime_properties(): raise RuntimeError('No cloud_resources node instances found.') runtime_properties = node_instance_runtime_properties( node_instance['id']) - logger.info('Runtime properties: {runtime_properties}'.format( - runtime_properties=runtime_properties)) if not runtime_properties: raise RuntimeError('No cloud_resources runtime_properties found.') return runtime_properties @@ -128,9 +138,18 @@ def cloud_resources_node_instance_runtime_properties(): def change_a_resource(props): group = props['resources']['example_security_group'] sg_id = group['instances'][0]['attributes']['id'] - environ['AWS_DEFAULT_REGION'] = \ - props['resource_config']['variables']['aws_region'] - ec2 = client('ec2') + terraform_vars = props['resource_config']['variables'] + environ['AWS_DEFAULT_REGION'] = terraform_vars['aws_region'] + access = get_secret(terraform_vars['access_key']) + secret = get_secret(terraform_vars['secret_key']) + client_kwargs = dict( + aws_access_key_id=access, + aws_secret_access_key=secret, + ) + if 'token' in terraform_vars: + token = get_secret(terraform_vars['token']) + client_kwargs.update({'aws_session_token': token}) + ec2 = client('ec2', **client_kwargs) ec2.authorize_security_group_ingress( GroupId=sg_id, IpProtocol="tcp", @@ -138,3 +157,13 @@ def change_a_resource(props): FromPort=53, ToPort=53 ) + + +def get_secret(value): + try: + loaded_value = loads(value) + except JSONDecodeError: + return value + secret_name = loaded_value['get_secret'] + value = cloudify_exec('cfy secrets get {}'.format(secret_name), log=False) + return value.get('value') diff --git a/CHANGELOG.txt b/CHANGELOG.txt index aa3db1b..f3b6fc5 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,6 @@ +0.20.0: + - CYBL-2014 Support backend migration from local to hosted. + - add __version__.py file in cloudify_tf folder 0.19.15: Do not delete files when debug node used. 0.19.14: - Fix check status and check drift diff --git a/cloudify_tf/__version__.py b/cloudify_tf/__version__.py new file mode 100644 index 0000000..2da632a --- /dev/null +++ b/cloudify_tf/__version__.py @@ -0,0 +1 @@ +version = '0.20.0' diff --git a/cloudify_tf/tasks.py b/cloudify_tf/tasks.py index 18995ea..d301b2b 100644 --- a/cloudify_tf/tasks.py +++ b/cloudify_tf/tasks.py @@ -16,6 +16,7 @@ import os import sys +from deepdiff import DeepDiff from cloudify.decorators import operation from cloudify import ctx as ctx_from_imports from cloudify.utils import exception_to_error_cause @@ -173,14 +174,35 @@ def apply(ctx, tf, force=False, **kwargs): _apply(tf, old_plan, force) +@operation +@with_terraform +def migrate_state(ctx, tf, backend, backend_config, **_): + name = backend.get('name') + options = backend.get('options') + credentials = backend.get('credentials', {}) + if credentials: + ctx.logger.info('Credentials are not used in migrate-state.') + tf.migrate_state(name, options, backend_config) + resource_config = utils.get_resource_config() + resource_config.update({'backend': backend}) + utils.update_resource_config(resource_config) + + class FailedPlanValidation(NonRecoverableError): pass -def compare_plan_results(new_plan, old_plan, force): - if old_plan != new_plan: - ctx_from_imports.logger.debug('New plan and old plan diff {}'.format( - set(old_plan) ^ set(new_plan))) +def compare_plan_results(new_plan, old_plan): + + left = sorted(old_plan.get('resource_changes', []), + key=lambda d: d['address']) + right = sorted(new_plan.get('resource_changes', []), + key=lambda d: d['address']) + + diff = DeepDiff(left, right) + if diff: + ctx_from_imports.logger.info( + 'Old plan and new plan diff {}'.format(diff)) raise FailedPlanValidation( 'The new plan differs from the old plan. ' 'Please Rerun plan workflow before executing apply worfklow.') @@ -193,7 +215,7 @@ def _apply(tf, old_plan=None, force=False): tf.run_terratag() if old_plan and not force: new_plan = tf.plan_and_show() - compare_plan_results(new_plan, old_plan, force) + compare_plan_results(new_plan, old_plan) if not force: tf.check_tflint() tf.check_tfsec() diff --git a/cloudify_tf/terraform/__init__.py b/cloudify_tf/terraform/__init__.py index ae49291..02540fc 100644 --- a/cloudify_tf/terraform/__init__.py +++ b/cloudify_tf/terraform/__init__.py @@ -310,8 +310,7 @@ def plan_file(self): json_result, _ = self.plan_and_show_two_formats() with tempfile.NamedTemporaryFile( 'w', - suffix='.json', - delete=delete_debug()) as plan_file: + suffix='.json') as plan_file: plan_file.write(json.dumps(json_result)) yield plan_file.name @@ -389,8 +388,10 @@ def tfvars(self): def tfvars(self, value): self._tfvars = value - def init(self, command_line_args=None): - cmdline = ['init', '-no-color', '-input=false'] + def init(self, command_line_args=None, prefix=None, no_input=True): + cmdline = ['init', '-no-color'] + if no_input: + cmdline.append('-input=false') if self.plugins_dir: cmdline.append('--plugin-dir=%s' % self.plugins_dir) if self.provider_upgrade: @@ -398,9 +399,26 @@ def init(self, command_line_args=None): command = self._tf_command(cmdline) if command_line_args: command.extend(command_line_args) + if prefix: + command[:0] = prefix with self.runtime_file(command): return self.execute(command) + def migrate_state(self, name, options, backend_config): + migrate_args = [] + answer_yes = ['echo', 'yes', '|'] + self._backend = { + 'name': name, + 'options': options, + } + self.put_backend() + for key, value in backend_config.items(): + migrate_args.append( + '-backend-config="{key}={value}"'.format( + key=key, value=value)) + migrate_args.append('-migrate-state') + self.init(migrate_args, answer_yes, no_input=False) + def destroy(self): command = self._tf_command(['destroy', '-auto-approve', @@ -493,7 +511,7 @@ def plan_and_show_two_formats(self): Execute terraform plan, then terraform show on the generated tfplan file """ - with tempfile.NamedTemporaryFile(delete=delete_debug()) as plan_file: + with tempfile.NamedTemporaryFile() as plan_file: self.plan(plan_file.name) json_result = self.show(plan_file.name) plain_text_result = self.show_plain_text(plan_file.name) @@ -505,7 +523,7 @@ def plan_and_show_state(self): then terraform show on the generated tfplan file """ status_problems = [] - with tempfile.NamedTemporaryFile(delete=delete_debug()) as plan_file: + with tempfile.NamedTemporaryFile() as plan_file: self.plan(plan_file.name) plan = self.show(plan_file.name) self.refresh() diff --git a/cloudify_tf/tests/test_plugin.py b/cloudify_tf/tests/test_plugin.py index a32e3f0..498ed8d 100644 --- a/cloudify_tf/tests/test_plugin.py +++ b/cloudify_tf/tests/test_plugin.py @@ -30,6 +30,7 @@ from ..tasks import (apply, install, check_drift, + migrate_state, setup_linters, import_resource, set_directory_config) @@ -683,3 +684,61 @@ def test_import_resource(self, mock_resource_config, *_): {'example_vm': tf_pulled_resources.get('resources')[0]}) self.assertEqual(ctx.instance.runtime_properties['outputs'], tf_output) + + @patch('cloudify_tf.terraform.utils.get_binary_location_from_rel') + @patch('cloudify_tf.terraform.Terraform.runtime_file') + @patch('cloudify_tf.terraform.Terraform.version') + @patch('cloudify_tf.decorators.get_terraform_source') + @patch('cloudify_common_sdk.utils.get_deployment_dir') + @patch('cloudify_tf.utils.get_plugins_dir') + @patch('cloudify_tf.utils.dump_file') + @patch('cloudify_tf.utils.store_sensitive_properties') + @patch('cloudify_tf.utils._unzip_archive') + @patch('cloudify_tf.utils.copy_directory') + @patch('cloudify_tf.utils.get_terraform_state_file', return_value=False) + @patch('cloudify_tf.utils.get_cloudify_version', return_value="6.1.0") + @patch('cloudify_tf.utils.get_node_instance_dir', + return_value=test_dir3) + @patch('cloudify_tf.terraform.Terraform.terraform_outdated', + return_value=False) + @patch('cloudify_tf.utils.store_sensitive_properties') + @patch('cloudify_tf.terraform.Terraform.set_plugins_dir') + @patch('cloudify_tf.utils.get_executable_path') + @patch('cloudify_tf.terraform.Terraform.execute') + @patch('cloudify_tf.utils.get_resource_config') + def test_migrate_state(self, + mock_resource_config, + mock_execute, + mock_exec_path, + mock_plugins_dir, + *_): + conf = self.get_terraform_module_conf_props(test_dir3) + mock_resource_config.return_value = conf.get('resource_config') + mock_exec_path.return_value = 'terraform' + mock_plugins_dir.return_value = 'foo' + ctx = self.mock_ctx("test_migrate_state", conf) + current_ctx.set(ctx=ctx) + backend = { + 'name': 'foo', + 'options': { + 'bar': 'baz' + } + } + backend_config = { + 'bar': 'baz' + } + kwargs = dict(ctx=ctx, + backend=backend, + backend_config=backend_config) + migrate_state(**kwargs) + mock_execute.assert_called_with([ + 'echo', + 'yes', + '|', + 'terraform', + 'init', + '-no-color', + "--plugin-dir=foo", + '-backend-config="bar=baz"', + '-migrate-state'] + ) diff --git a/cloudify_tf/tests/test_workflows.py b/cloudify_tf/tests/test_workflows.py index 89d38e2..426bddf 100644 --- a/cloudify_tf/tests/test_workflows.py +++ b/cloudify_tf/tests/test_workflows.py @@ -1,11 +1,18 @@ -from mock import MagicMock +from mock import patch, MagicMock from unittest import TestCase from cloudify.mocks import MockContext from cloudify.state import current_ctx +from cloudify.exceptions import NonRecoverableError from .. import workflows +class MockedWorkflowCtx(MockContext): + + def graph_mode(self): + return MagicMock() + + class TFWorkflowTests(TestCase): @staticmethod @@ -47,3 +54,54 @@ def test_plan_module_instance(self): sequence = self._mock_sequence() workflows._plan_module_instance(ctx, node, instance, sequence, {}) assert sequence.add.call_count == 4 + + def test_migrate_state(self): + ctx = MockedWorkflowCtx() + current_ctx.set(ctx) + backend = {} + backend_config = {'baz': 'qux'} + nodes = ['foo'] + node_instances = [] + with self.assertRaisesRegex( + NonRecoverableError, 'No new backend was provided'): + workflows.migrate_state( + ctx, + node_ids=nodes, + node_instance_ids=node_instances, + backend=backend, + backend_config=backend_config + ) + backend = {'foo': 'bar'} + nodes = ['foo'] + node_instances = ['bar'] + with self.assertRaisesRegex( + NonRecoverableError, 'mutually exclusive'): + workflows.migrate_state( + ctx, + node_ids=nodes, + node_instance_ids=node_instances, + backend=backend, + backend_config=backend_config + ) + with patch('cloudify_tf.workflows._terraform_operation') as op: + backend = {'foo': 'bar'} + nodes = ['foo'] + node_instances = [] + kwargs = { + 'backend': backend, + 'backend_config': backend_config + } + workflows.migrate_state( + ctx, + node_ids=nodes, + node_instance_ids=node_instances, + backend=backend, + backend_config=backend_config + ) + op.assert_called_once_with( + ctx, + "terraform.migrate_state", + nodes, + node_instances, + **kwargs + ) diff --git a/cloudify_tf/utils.py b/cloudify_tf/utils.py index 25bde5f..26b8a18 100644 --- a/cloudify_tf/utils.py +++ b/cloudify_tf/utils.py @@ -38,6 +38,7 @@ from cloudify_common_sdk.hcl import ( convert_json_hcl, extract_hcl_from_dict, + remove_quotes_from_vars, ) from cloudify_common_sdk.utils import ( v1_gteq_v2, @@ -820,7 +821,9 @@ def create_backend_string(name, options): 'option_value': options } )) - return 'terraform {{\n{}}}'.format(indent(backend_block, ' ')) + terraform_block = 'terraform {{\n{}}}'.format( + indent(backend_block, ' ')) + return remove_quotes_from_vars(terraform_block) def create_required_providers_string(items): diff --git a/cloudify_tf/workflows.py b/cloudify_tf/workflows.py index 76e45d6..9670f2d 100644 --- a/cloudify_tf/workflows.py +++ b/cloudify_tf/workflows.py @@ -25,8 +25,15 @@ PRECONFIGURE = 'cloudify.interfaces.relationship_lifecycle.preconfigure' -def _terraform_operation(ctx, operation, node_ids, - node_instance_ids, **kwargs): +def _terraform_operation(ctx, + operation, + node_ids, + node_instance_ids, + **kwargs): + if 'force' in kwargs: + force = kwargs.get('force') + kwargs['force'] = eval(force) + graph = ctx.graph_mode() sequence = graph.sequence() # Iterate over all node instances of type "cloudify.nodes.terraform.Module" @@ -139,6 +146,28 @@ def run_infracost(ctx, **kwargs).execute() +def migrate_state(ctx, node_ids=None, node_instance_ids=None, **kwargs): + graph = ctx.graph_mode() + + if not kwargs.get('backend'): + raise NonRecoverableError('No new backend was provided.') + + if node_ids and node_instance_ids: + raise NonRecoverableError( + 'The parameters node_ids and node_instance_ids are ' + 'mutually exclusive. ' + '{} and {} were provided.'.format(node_ids, node_instance_ids) + ) + _terraform_operation( + ctx, + "terraform.migrate_state", + node_ids, + node_instance_ids, + **kwargs).execute() + + return graph.execute() + + def terraform_plan(ctx, node_ids=None, node_instance_ids=None, diff --git a/examples/blueprint-examples b/examples/blueprint-examples index 8c7d4cf..bfa978f 160000 --- a/examples/blueprint-examples +++ b/examples/blueprint-examples @@ -1 +1 @@ -Subproject commit 8c7d4cf9821b87a80263dcf9cb9fd2bfe0498547 +Subproject commit bfa978ff7805a3545bf0e7dccf2988179148567f diff --git a/ignore_plugin_yaml_differences b/ignore_plugin_yaml_differences index f02d394..21cdc9c 100644 --- a/ignore_plugin_yaml_differences +++ b/ignore_plugin_yaml_differences @@ -1 +1 @@ -{'dictionary_item_added': [root['workflows']['refresh_terraform_resources']['parameters']['node_instance_ids']['type'], root['workflows']['refresh_terraform_resources']['parameters']['node_ids']['type'], root['workflows']['terraform_plan']['parameters']['node_instance_ids']['type'], root['workflows']['terraform_plan']['parameters']['node_ids']['type'], root['workflows']['reload_terraform_template']['parameters']['node_instance_ids']['type'], root['workflows']['reload_terraform_template']['parameters']['node_ids']['type'], root['workflows']['update_terraform_binary']['parameters']['node_instance_ids']['type'], root['workflows']['update_terraform_binary']['parameters']['node_ids']['type'], root['workflows']['import_terraform_resource']['parameters']['node_instance_ids']['type'], root['workflows']['import_terraform_resource']['parameters']['node_ids']['type'], root['workflows']['run_infracost']['parameters']['node_instance_ids']['type'], root['workflows']['run_infracost']['parameters']['node_ids']['type']], 'dictionary_item_removed': [root['node_types']['cloudify.nodes.terraform.Module']['interfaces']['cloudify.interfaces.lifecycle']['update']['inputs']], 'values_changed': {"root['node_types']['cloudify.nodes.terraform.Module']['interfaces']['cloudify.interfaces.lifecycle']['update']['implementation']": {'new_value': 'tf.cloudify_tf.tasks.update', 'old_value': 'tf.cloudify_tf.tasks.reload_template'}}} +{'dictionary_item_added': [root['workflows']['refresh_terraform_resources']['parameters']['node_instance_ids']['type'], root['workflows']['refresh_terraform_resources']['parameters']['node_ids']['type'], root['workflows']['terraform_plan']['parameters']['node_instance_ids']['type'], root['workflows']['terraform_plan']['parameters']['node_ids']['type'], root['workflows']['reload_terraform_template']['parameters']['node_instance_ids']['type'], root['workflows']['reload_terraform_template']['parameters']['node_ids']['type'], root['workflows']['update_terraform_binary']['parameters']['node_instance_ids']['type'], root['workflows']['update_terraform_binary']['parameters']['node_ids']['type'], root['workflows']['import_terraform_resource']['parameters']['node_instance_ids']['type'], root['workflows']['import_terraform_resource']['parameters']['node_ids']['type'], root['workflows']['run_infracost']['parameters']['node_instance_ids']['type'], root['workflows']['run_infracost']['parameters']['node_ids']['type'], root['workflows']['migrate_state']['parameters']['node_instance_ids']['type'], root['workflows']['migrate_state']['parameters']['node_ids']['type']]} diff --git a/plugin.yaml b/plugin.yaml index 927f0a0..31f0c54 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2,7 +2,7 @@ plugins: tf: executor: central_deployment_agent package_name: cloudify-terraform-plugin - package_version: '0.19.15' + package_version: '0.20.0' dsl_definitions: @@ -49,7 +49,7 @@ data_types: default: false installation_source: type: string - default: 'https://releases.hashicorp.com/terraform/1.1.4/terraform_1.1.4_linux_amd64.zip' + default: 'https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip' description: Location to download the Terraform executable binary from. Ignored if 'use_existing' is true. plugins: # type: list commented for 4.X support @@ -526,6 +526,14 @@ node_types: default: "terraform/deny" opa_config: default: { get_property: [SELF, opa_config ] } + migrate_state: + implementation: tf.cloudify_tf.tasks.migrate_state + inputs: + backend: + default: { get_property: [ SELF, resource_config, backend ] } + backend_config: + type: dict + default: {} relationships: @@ -618,3 +626,43 @@ workflows: Note about enable property it is not considered here as it is enabled by default type: cloudify.types.terraform.infracost default: {} + + migrate_state: + # Usage: cfy executions start migrate_state -d [DEPLOYMENT_ID] -p params.yaml + ########################################################################## + # # params.yaml: + # node_ids: + # - cloud_resources + # backend: + # name: s3 + # options: + # bucket: cybl-2014 + # key: t2 + # region: var.aws_region + # access_key: var.access_key + # secret_key: var.secret_key + # backend_config: + # bucket: cybl-2014 + # key: t2 + # region: us-east-2 + # access_key: { get_secret: aws_access_key_id } + # secret_key: { get_secret: aws_secret_access_key } + mapping: tf.cloudify_tf.workflows.migrate_state + parameters: + <<: *terraform_workflow_params + backend: + description: > + A backend config as may be provided in the node template properties. + For example, for azurerm: + name: azurerm + options: + storage_account_name: mysa + container_name: mycontainer + key: mykey.tfstate + resource_group_name: myrg + type: cloudify.types.terraform.Backend + backend_config: + type: dict + description: | + Key, value pairs that will be passed to init as the args terraform init -backend-config="key=value". + default: {} diff --git a/plugin_1_4.yaml b/plugin_1_4.yaml index 26d95ce..4747adb 100644 --- a/plugin_1_4.yaml +++ b/plugin_1_4.yaml @@ -2,7 +2,7 @@ plugins: tf: executor: central_deployment_agent package_name: cloudify-terraform-plugin - package_version: '0.19.15' + package_version: '0.20.0' dsl_definitions: @@ -49,7 +49,7 @@ data_types: default: false installation_source: type: string - default: 'https://releases.hashicorp.com/terraform/1.1.4/terraform_1.1.4_linux_amd64.zip' + default: 'https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip' description: Location to download the Terraform executable binary from. Ignored if 'use_existing' is true. plugins: # type: list commented for 4.X support @@ -165,6 +165,16 @@ data_types: type: string description: file Name .tfvars required: false + store_output_secrets: + type: dict + description: | + add/update secrets from terraform senstive outputs. + should be in this format + terraform_output_name: secrets_to_created_or_updated + required: false + obfuscate_sensitive: + type: boolean + default: false cloudify.types.terraform.tfsec: @@ -409,8 +419,34 @@ node_types: # read from the last location it was loaded from. # This can be overridden by specifying the 'source' input. implementation: tf.cloudify_tf.tasks.reload_template - update: - implementation: tf.cloudify_tf.tasks.update + update: &terraform_reload + <<: *terraform_heal + inputs: + source: + description: > + URL or path to a ZIP/tar.gz file or a Git repository to obtain + new module source from. If omitted, then the module is reloaded + from its last location. + type: string + default: { get_attribute: [ SELF, last_source_location ] } + source_path: + type: string + description: The path within the source property, where the terraform files may be found. + default: { get_property: [ SELF, resource_config, source_path ] } + variables: + type: dict + description: Variables override. + default: { get_property: [ SELF, resource_config, variables ] } + environment_variables: + type: dict + description: Environment variables override. + default: { get_property: [ SELF, resource_config, environment_variables ] } + destroy_previous: + description: > + If true, then the plugin destroys the existing Terraform + topology before applying the new one. + type: boolean + default: false check_drift: implementation: tf.cloudify_tf.tasks.check_drift terraform: @@ -441,33 +477,7 @@ node_types: terratag_config: default: { get_property: [ SELF, terratag_config ] } reload: - <<: *terraform_heal - inputs: - source: - description: > - URL or path to a ZIP/tar.gz file or a Git repository to obtain - new module source from. If omitted, then the module is reloaded - from its last location. - type: string - default: { get_attribute: [ SELF, last_source_location ] } - source_path: - type: string - description: The path within the source property, where the terraform files may be found. - default: { get_property: [ SELF, resource_config, source_path ] } - variables: - type: dict - description: Variables override. - default: { get_property: [ SELF, resource_config, variables ] } - environment_variables: - type: dict - description: Environment variables override. - default: { get_property: [ SELF, resource_config, environment_variables ] } - destroy_previous: - description: > - If true, then the plugin destroys the existing Terraform - topology before applying the new one. - type: boolean - default: false + <<: *terraform_reload refresh: # Refreshes Terraform's state. implementation: tf.cloudify_tf.tasks.state_pull @@ -516,6 +526,14 @@ node_types: default: "terraform/deny" opa_config: default: { get_property: [SELF, opa_config ] } + migrate_state: + implementation: tf.cloudify_tf.tasks.migrate_state + inputs: + backend: + default: { get_property: [ SELF, resource_config, backend ] } + backend_config: + type: dict + default: {} relationships: @@ -629,6 +647,45 @@ workflows: type: cloudify.types.terraform.infracost default: {} + migrate_state: + # Usage: cfy executions start migrate_state -d [DEPLOYMENT_ID] -p params.yaml + ########################################################################## + # # params.yaml: + # node_ids: + # - cloud_resources + # backend: + # name: s3 + # options: + # bucket: cybl-2014 + # key: t2 + # region: var.aws_region + # access_key: var.access_key + # secret_key: var.secret_key + # backend_config: + # bucket: cybl-2014 + # key: t2 + # region: us-east-2 + # access_key: { get_secret: aws_access_key_id } + # secret_key: { get_secret: aws_secret_access_key } + mapping: tf.cloudify_tf.workflows.migrate_state + parameters: + <<: *terraform_workflow_params + backend: + description: > + A backend config as may be provided in the node template properties. + For example, for azurerm: + name: azurerm + options: + storage_account_name: mysa + container_name: mycontainer + key: mykey.tfstate + resource_group_name: myrg + type: cloudify.types.terraform.Backend + backend_config: + type: dict + description: | + Key, value pairs that will be passed to init as the args terraform init -backend-config="key=value". + default: {} blueprint_labels: obj-type: diff --git a/setup.py b/setup.py index 22463a7..cece886 100644 --- a/setup.py +++ b/setup.py @@ -15,24 +15,17 @@ import os +import re +import pathlib from setuptools import setup -def read(rel_path): - here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, rel_path), 'r') as fp: - return fp.read() - - -def get_version(rel_file='plugin.yaml'): - lines = read(rel_file) - for line in lines.splitlines(): - if 'package_version' in line: - split_line = line.split(':') - line_no_space = split_line[-1].replace(' ', '') - line_no_quotes = line_no_space.replace('\'', '') - return line_no_quotes.strip('\n') - raise RuntimeError('Unable to find version string.') +def get_version(): + current_dir = pathlib.Path(__file__).parent.resolve() + with open(os.path.join(current_dir,'cloudify_tf/__version__.py'), + 'r') as outfile: + var = outfile.read() + return re.search(r'\d+.\d+.\d+', var).group() setup( @@ -48,6 +41,7 @@ def get_version(rel_file='plugin.yaml'): "cloudify-common>=4.5.5", "networkx==1.9.1", "requests>=2.7.0,<3.0", - "cloudify-utilities-plugins-sdk>=0.0.104", + "cloudify-utilities-plugins-sdk>=0.0.114", + 'deepdiff==3.3.0', ] ) diff --git a/test-requirements.txt b/test-requirements.txt index f93bb4c..de64231 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,4 +9,3 @@ flake8 # For integration tests pytest==4.6.3 pyyaml>=4.2b1 -cloudify-utilities-plugins-sdk>=0.0.93 diff --git a/v2_plugin.yaml b/v2_plugin.yaml index ef5b708..c3af95b 100644 --- a/v2_plugin.yaml +++ b/v2_plugin.yaml @@ -2,7 +2,7 @@ plugins: tf: executor: central_deployment_agent package_name: cloudify-terraform-plugin - package_version: '0.19.15' + package_version: '0.20.0' dsl_definitions: @@ -49,7 +49,7 @@ data_types: default: false installation_source: type: string - default: 'https://releases.hashicorp.com/terraform/1.1.4/terraform_1.1.4_linux_amd64.zip' + default: 'https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip' description: Location to download the Terraform executable binary from. Ignored if 'use_existing' is true. plugins: # type: list commented for 4.X support @@ -419,8 +419,34 @@ node_types: # read from the last location it was loaded from. # This can be overridden by specifying the 'source' input. implementation: tf.cloudify_tf.tasks.reload_template - update: - implementation: tf.cloudify_tf.tasks.update + update: &terraform_reload + <<: *terraform_heal + inputs: + source: + description: > + URL or path to a ZIP/tar.gz file or a Git repository to obtain + new module source from. If omitted, then the module is reloaded + from its last location. + type: string + default: { get_attribute: [ SELF, last_source_location ] } + source_path: + type: string + description: The path within the source property, where the terraform files may be found. + default: { get_property: [ SELF, resource_config, source_path ] } + variables: + type: dict + description: Variables override. + default: { get_property: [ SELF, resource_config, variables ] } + environment_variables: + type: dict + description: Environment variables override. + default: { get_property: [ SELF, resource_config, environment_variables ] } + destroy_previous: + description: > + If true, then the plugin destroys the existing Terraform + topology before applying the new one. + type: boolean + default: false check_drift: implementation: tf.cloudify_tf.tasks.check_drift terraform: @@ -451,33 +477,7 @@ node_types: terratag_config: default: { get_property: [ SELF, terratag_config ] } reload: - <<: *terraform_heal - inputs: - source: - description: > - URL or path to a ZIP/tar.gz file or a Git repository to obtain - new module source from. If omitted, then the module is reloaded - from its last location. - type: string - default: { get_attribute: [ SELF, last_source_location ] } - source_path: - type: string - description: The path within the source property, where the terraform files may be found. - default: { get_property: [ SELF, resource_config, source_path ] } - variables: - type: dict - description: Variables override. - default: { get_property: [ SELF, resource_config, variables ] } - environment_variables: - type: dict - description: Environment variables override. - default: { get_property: [ SELF, resource_config, environment_variables ] } - destroy_previous: - description: > - If true, then the plugin destroys the existing Terraform - topology before applying the new one. - type: boolean - default: false + <<: *terraform_reload refresh: # Refreshes Terraform's state. implementation: tf.cloudify_tf.tasks.state_pull @@ -526,6 +526,14 @@ node_types: default: "terraform/deny" opa_config: default: { get_property: [SELF, opa_config ] } + migrate_state: + implementation: tf.cloudify_tf.tasks.migrate_state + inputs: + backend: + default: { get_property: [ SELF, resource_config, backend ] } + backend_config: + type: dict + default: {} relationships: @@ -619,6 +627,46 @@ workflows: type: cloudify.types.terraform.infracost default: {} + migrate_state: + # Usage: cfy executions start migrate_state -d [DEPLOYMENT_ID] -p params.yaml + ########################################################################## + # # params.yaml: + # node_ids: + # - cloud_resources + # backend: + # name: s3 + # options: + # bucket: cybl-2014 + # key: t2 + # region: var.aws_region + # access_key: var.access_key + # secret_key: var.secret_key + # backend_config: + # bucket: cybl-2014 + # key: t2 + # region: us-east-2 + # access_key: { get_secret: aws_access_key_id } + # secret_key: { get_secret: aws_secret_access_key } + mapping: tf.cloudify_tf.workflows.migrate_state + parameters: + <<: *terraform_workflow_params + backend: + description: > + A backend config as may be provided in the node template properties. + For example, for azurerm: + name: azurerm + options: + storage_account_name: mysa + container_name: mycontainer + key: mykey.tfstate + resource_group_name: myrg + type: cloudify.types.terraform.Backend + backend_config: + type: dict + description: | + Key, value pairs that will be passed to init as the args terraform init -backend-config="key=value". + default: {} + blueprint_labels: obj-type: values: