diff --git a/.circleci/update_test_manager.py b/.circleci/update_test_manager.py new file mode 100644 index 0000000..3e5131a --- /dev/null +++ b/.circleci/update_test_manager.py @@ -0,0 +1,12 @@ +from os import path, pardir +from ecosystem_tests.dorkl import replace_plugin_package_on_manager +from ecosystem_cicd_tools.validations import validate_plugin_version + +abs_path = path.join( + path.abspath(path.join(path.dirname(__file__), pardir))) + +if __name__ == '__main__': + version = validate_plugin_version(abs_path) + for package in ['cloudify_tf']: + replace_plugin_package_on_manager( + 'cloudify-terraform-plugin', version, package, ) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 53c9fb8..0345782 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,7 @@ +0.14.4: + - Handle existing binary on the manager. + - Mask AWS credentials if provided through environment variables. + - Some logging improvements. 0.14.3: - Update PyYAML Security vulnerability. 0.14.2: diff --git a/cloudify_tf/tasks.py b/cloudify_tf/tasks.py index 9547ad9..5f74108 100644 --- a/cloudify_tf/tasks.py +++ b/cloudify_tf/tasks.py @@ -128,15 +128,8 @@ def install(ctx, **_): 'If you do not have sufficient permissions, that ' 'installation will fail.'.format( loc=executable_path)) - installation_zip = os.path.join(installation_dir, 'tf.zip') - ctx.logger.info( - 'Downloading Terraform from {source} into {zip}.'.format( - source=installation_source, - zip=installation_zip)) - utils.download_file(installation_zip, installation_source) - executable_dir = os.path.dirname(executable_path) - utils._unzip_and_set_permissions(installation_zip, executable_dir) - os.remove(installation_zip) + utils.install_binary( + installation_dir, executable_path, installation_source) # store the values in the runtime for safe keeping -> validation ctx.instance.runtime_properties['executable_path'] = executable_path @@ -151,50 +144,59 @@ def uninstall(ctx, **_): exc_path = terraform_config.get('executable_path', '') system_exc = resource_config.get('use_existing_resource') - if os.path.isfile(exc_path) and not system_exc: - ctx.logger.info('Removing executable: {path}'.format(path=exc_path)) - os.remove(exc_path) - elif os.path.isfile(exc_path): - ctx.logger.warn('Unable to remove file {loc} because it is a system ' - 'resource.'.format(loc=exc_path)) + if os.path.isfile(exc_path): + if system_exc: + ctx.logger.info( + 'Not removing Terraform installation at {loc} as' + 'it was provided externally'.format(loc=exc_path)) + else: + ctx.logger.info('Removing executable: {path}'.format( + path=exc_path)) + os.remove(exc_path) for property_name, property_desc in [ ('plugins_dir', 'plugins directory'), ('storage_path', 'storage_directory')]: - dir_to_delete = terraform_config.get(property_name, '') - utils.remove_dir(dir_to_delete, property_desc) + dir_to_delete = terraform_config.get(property_name, None) + if dir_to_delete: + utils.remove_dir(dir_to_delete, property_desc) @operation -@skip_if_existing def set_directory_config(ctx, **_): exc_path = utils.get_executable_path(target=True) plugins_dir = utils.get_plugins_dir(target=True) storage_path = utils.get_storage_path(target=True) + deployment_terraform_dir = os.path.join(storage_path, + '.terraform') resource_node_instance_dir = utils.get_node_instance_dir(source=True) if not os.path.exists(resource_node_instance_dir): mkdir_p(resource_node_instance_dir) resource_terraform_dir = os.path.join(resource_node_instance_dir, '.terraform') - deployment_terraform_dir = os.path.join(storage_path, - '.terraform') - - # We don't want to put all the plugins for all the node instances in a - # deployment multiple times on the system. So here, - # we already stored it once on the file system, and now we create - # symlinks so other deployments can use it. - # TODO: Possibly put this in "apply" and remove the relationship in - # the future. - - ctx.logger.info('Creating link {src} {dst}'.format( - src=deployment_terraform_dir, dst=resource_terraform_dir)) - os.symlink(deployment_terraform_dir, resource_terraform_dir) - resource_plugins_dir = plugins_dir.replace( ctx.target.instance.id, ctx.source.instance.id) resource_storage_dir = storage_path.replace( ctx.target.instance.id, ctx.source.instance.id) + if utils.is_using_existing(target=True): + # We are going to use a TF binary at another location. + # However, we still need to make sure that this directory exists. + # Otherwise TF will complain. It does not create it. + # In our other scenario, a symlink is created. + mkdir_p(resource_terraform_dir) + else: + # We don't want to put all the plugins for all the node instances in a + # deployment multiple times on the system. So here, + # we already stored it once on the file system, and now we create + # symlinks so other deployments can use it. + # TODO: Possibly put this in "apply" and remove the relationship in + # the future. + + ctx.logger.info('Creating link {src} {dst}'.format( + src=deployment_terraform_dir, dst=resource_terraform_dir)) + os.symlink(deployment_terraform_dir, resource_terraform_dir) + ctx.logger.info("setting executable_path to {path}".format( path=exc_path)) ctx.logger.info("setting plugins_dir to {dir}".format( diff --git a/cloudify_tf/terraform/__init__.py b/cloudify_tf/terraform/__init__.py index 9bc9534..df054ed 100644 --- a/cloudify_tf/terraform/__init__.py +++ b/cloudify_tf/terraform/__init__.py @@ -19,13 +19,7 @@ from contextlib import contextmanager -from cloudify.exceptions import NonRecoverableError - -from ..utils import ( - run_subprocess, - get_plugins_dir, - get_executable_path, - get_resource_config) +from .. import utils class Terraform(object): @@ -67,7 +61,7 @@ def set_plugins_dir(path): return path def execute(self, command, return_output=False): - return run_subprocess( + return utils.run_subprocess( command, self.logger, self.root_module, self.env, return_output=return_output) @@ -135,14 +129,12 @@ def refresh(self): @staticmethod def from_ctx(ctx, terraform_source): - executable_path = get_executable_path() - plugins_dir = get_plugins_dir() - resource_config = get_resource_config() - if not os.path.exists(executable_path): - raise NonRecoverableError( - "Terraform's executable not found in {0}. Please set the " - "'executable_path' property accordingly.".format( - executable_path)) + executable_path = utils.get_executable_path() or \ + utils.get_binary_location_from_rel() + plugins_dir = utils.get_plugins_dir() + resource_config = utils.get_resource_config() + if not os.path.exists(plugins_dir) and utils.is_using_existing(): + utils.mkdir_p(plugins_dir) env_variables = resource_config.get('environment_variables') tf = Terraform( ctx.logger, diff --git a/cloudify_tf/utils.py b/cloudify_tf/utils.py index 762677a..b3f9293 100644 --- a/cloudify_tf/utils.py +++ b/cloudify_tf/utils.py @@ -45,6 +45,11 @@ TERRAFORM_STATE_FILE = 'terraform.tfstate' +MASKED_ENV_VARS = { + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' +} + def download_file(source, destination): run_subprocess(['curl', '-o', source, destination]) @@ -71,12 +76,18 @@ def run_subprocess(command, passed_env.update(os.environ) passed_env.update(additional_env) + printed_args = copy.deepcopy(args_to_pass) + printed_env = printed_args.get('env', {}) + for env_var in printed_env.keys(): + if env_var in MASKED_ENV_VARS: + printed_env[env_var] = '****' + logger.info('Running: command={cmd}, ' 'cwd={cwd}, ' 'additional_args={args}'.format( cmd=command, cwd=cwd, - args=args_to_pass)) + args=printed_args)) process = subprocess.Popen( args=command, @@ -115,6 +126,8 @@ def exclude_file(dirname, filename, excluded_files): """ rel_path = os.path.join(dirname, filename) for f in excluded_files: + if not f: + continue if os.path.isfile(f) and rel_path == f: return True return False @@ -127,6 +140,8 @@ def exclude_dirs(dirname, subdirs, excluded_files): """ rel_subdirs = [os.path.join(dirname, d) for d in subdirs] for f in excluded_files: + if not f: + continue if os.path.isdir(f) and f in rel_subdirs: subdirs.remove(ntpath.basename(f)) @@ -220,7 +235,14 @@ def _create_source_path(source_tmp_path): return source_tmp_path -def _unzip_and_set_permissions(zip_file, target_dir): +def set_permissions(target_file): + run_subprocess( + ['chmod', 'u+x', target_file], + ctx.logger + ) + + +def unzip_and_set_permissions(zip_file, target_dir): """Unzip a file and fix permissions on the files.""" ctx.logger.debug('Unzipping into {dir}.'.format(dir=target_dir)) @@ -238,10 +260,7 @@ def _unzip_and_set_permissions(zip_file, target_dir): target_file = os.path.join(target_dir, name) ctx.logger.info('Setting executable permission on ' '{loc}.'.format(loc=target_file)) - run_subprocess( - ['chmod', 'u+x', target_file], - ctx.logger - ) + set_permissions(target_file) def get_instance(_ctx=None, target=False, source=False): @@ -271,9 +290,63 @@ def get_node(_ctx=None, target=False): def is_using_existing(target=True): """Decide if we need to do this work or not.""" resource_config = get_resource_config(target=target) + if not target: + tf_rel = find_terraform_node_from_rel() + if tf_rel: + resource_config = tf_rel.target.instance.runtime_properties.get( + 'resource_config', {}) return resource_config.get('use_existing_resource', True) +def get_binary_location_from_rel(): + tf_rel = find_terraform_node_from_rel() + terraform_config = tf_rel.target.node.properties.get( + 'terraform_config', {}) + candidate_a = terraform_config.get('executable_path') + candidate_b = get_executable_path() + if candidate_b and os.path.isfile(candidate_b): + return candidate_b + if candidate_a and os.path.isfile(candidate_a): + return candidate_a + raise NonRecoverableError( + "Terraform's executable not found in {0} or {1}. Please set the " + "'executable_path' property accordingly.".format( + candidate_b, candidate_a)) + + +def find_terraform_node_from_rel(): + return find_rel_by_type( + ctx.instance, 'cloudify.terraform.relationships.run_on_host') + + +def find_rel_by_type(node_instance, rel_type): + rels = find_rels_by_type(node_instance, rel_type) + return rels[0] if len(rels) > 0 else None + + +def find_rels_by_type(node_instance, rel_type): + return [x for x in node_instance.relationships + if rel_type in x.type_hierarchy] + + +def install_binary( + installation_dir, + executable_path, + installation_source=None): + + if installation_source: + installation_zip = os.path.join(installation_dir, 'tf.zip') + ctx.logger.info( + 'Downloading Terraform from {source} into {zip}.'.format( + source=installation_source, + zip=installation_zip)) + download_file(installation_zip, installation_source) + executable_dir = os.path.dirname(executable_path) + unzip_and_set_permissions(installation_zip, executable_dir) + os.remove(installation_zip) + return executable_path + + def get_resource_config(target=False): """Get the cloudify.nodes.terraform.Module resource_config""" instance = get_instance(target=target) @@ -312,7 +385,7 @@ def update_terraform_source_material(new_source, target=False): # Zip the file to store in runtime terraform_source_zip = _zip_archive(source_tmp_path) base64_rep = _file_to_base64(terraform_source_zip) - ctx.logger.warn('The before base64_rep size is {size}.'.format( + ctx.logger.info('The before base64_rep size is {size}.'.format( size=len(base64_rep))) instance.runtime_properties['terraform_source'] = base64_rep @@ -364,6 +437,11 @@ def get_executable_path(target=False): if not executable_path: executable_path = \ os.path.join(get_node_instance_dir(target=target), 'terraform') + if not os.path.exists(executable_path) and \ + is_using_existing(target=target): + node = get_node(target=target) + terraform_config = node.properties.get('terraform_config', {}) + executable_path = terraform_config.get('executable_path') instance.runtime_properties['executable_path'] = executable_path ctx.logger.debug('Value executable_path is {loc}.'.format( loc=executable_path)) @@ -447,6 +525,7 @@ def remove_dir(folder, desc=''): ctx.logger.info('Removing {desc}: {dir}'.format(desc=desc, dir=folder)) shutil.rmtree(folder) elif os.path.islink(folder): + ctx.logger.info('Unlinking: {}'.format(folder)) os.unlink(folder) else: ctx.logger.info( @@ -483,7 +562,7 @@ def handle_plugins(plugins, plugins_dir, installation_dir): download_file(plugin_zip.name, plugin_url) unzip_path = os.path.join(plugins_dir, plugin_name) mkdir_p(os.path.basename(unzip_path)) - _unzip_and_set_permissions(plugin_zip.name, unzip_path) + unzip_and_set_permissions(plugin_zip.name, unzip_path) os.remove(plugin_zip.name) diff --git a/plugin.yaml b/plugin.yaml index 3cfc17b..d0dfd94 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2,7 +2,7 @@ plugins: tf: executor: central_deployment_agent package_name: cloudify-terraform-plugin - package_version: '0.14.3' + package_version: '0.14.4' dsl_definitions: @@ -120,7 +120,7 @@ node_types: cloudify.interfaces.lifecycle: start: implementation: tf.cloudify_tf.tasks.apply - delete: + stop: implementation: tf.cloudify_tf.tasks.destroy terraform: reload: