Mask AWS credentials (#39)

* Mask AWS credentials
__NODOCS__

* move delete to stop (#40)

remove unused names

support external tf binary

Co-authored-by: EarthmanT <[email protected]>

.circleci/update_test_manager.py

@@ -0,0 +1,12 @@
+from os import path, pardir
+from ecosystem_tests.dorkl import replace_plugin_package_on_manager
+from ecosystem_cicd_tools.validations import validate_plugin_version
+
+abs_path = path.join(
+    path.abspath(path.join(path.dirname(__file__), pardir)))
+
+if __name__ == '__main__':
+    version = validate_plugin_version(abs_path)
+    for package in ['cloudify_tf']:
+        replace_plugin_package_on_manager(
+            'cloudify-terraform-plugin', version, package, )

CHANGELOG.txt

@@ -1,3 +1,7 @@
+0.14.4:
+    - Handle existing binary on the manager.
+    - Mask AWS credentials if provided through environment variables.
+    - Some logging improvements.
 0.14.3:
     - Update PyYAML Security vulnerability.
 0.14.2:

cloudify_tf/tasks.py

@@ -128,15 +128,8 @@ def install(ctx, **_):
                         'If you do not have sufficient permissions, that '
                         'installation will fail.'.format(
                             loc=executable_path))
-        installation_zip = os.path.join(installation_dir, 'tf.zip')
-        ctx.logger.info(
-            'Downloading Terraform from {source} into {zip}.'.format(
-                source=installation_source,
-                zip=installation_zip))
-        utils.download_file(installation_zip, installation_source)
-        executable_dir = os.path.dirname(executable_path)
-        utils._unzip_and_set_permissions(installation_zip, executable_dir)
-        os.remove(installation_zip)
+        utils.install_binary(
+            installation_dir, executable_path, installation_source)
 
     # store the values in the runtime for safe keeping -> validation
     ctx.instance.runtime_properties['executable_path'] = executable_path
@@ -151,50 +144,59 @@ def uninstall(ctx, **_):
     exc_path = terraform_config.get('executable_path', '')
     system_exc = resource_config.get('use_existing_resource')
 
-    if os.path.isfile(exc_path) and not system_exc:
-        ctx.logger.info('Removing executable: {path}'.format(path=exc_path))
-        os.remove(exc_path)
-    elif os.path.isfile(exc_path):
-        ctx.logger.warn('Unable to remove file {loc} because it is a system '
-                        'resource.'.format(loc=exc_path))
+    if os.path.isfile(exc_path):
+        if system_exc:
+            ctx.logger.info(
+                'Not removing Terraform installation at {loc} as'
+                'it was provided externally'.format(loc=exc_path))
+        else:
+            ctx.logger.info('Removing executable: {path}'.format(
+                path=exc_path))
+            os.remove(exc_path)
 
     for property_name, property_desc in [
             ('plugins_dir', 'plugins directory'),
             ('storage_path', 'storage_directory')]:
-        dir_to_delete = terraform_config.get(property_name, '')
-        utils.remove_dir(dir_to_delete, property_desc)
+        dir_to_delete = terraform_config.get(property_name, None)
+        if dir_to_delete:
+            utils.remove_dir(dir_to_delete, property_desc)
 
 
 @operation
-@skip_if_existing
 def set_directory_config(ctx, **_):
     exc_path = utils.get_executable_path(target=True)
     plugins_dir = utils.get_plugins_dir(target=True)
     storage_path = utils.get_storage_path(target=True)
+    deployment_terraform_dir = os.path.join(storage_path,
+                                            '.terraform')
     resource_node_instance_dir = utils.get_node_instance_dir(source=True)
     if not os.path.exists(resource_node_instance_dir):
         mkdir_p(resource_node_instance_dir)
     resource_terraform_dir = os.path.join(resource_node_instance_dir,
                                           '.terraform')
-    deployment_terraform_dir = os.path.join(storage_path,
-                                            '.terraform')
-
-    # We don't want to put all the plugins for all the node instances in a
-    # deployment multiple times on the system. So here,
-    # we already stored it once on the file system, and now we create
-    # symlinks so other deployments can use it.
-    # TODO: Possibly put this in "apply" and remove the relationship in
-    # the future.
-
-    ctx.logger.info('Creating link {src} {dst}'.format(
-        src=deployment_terraform_dir, dst=resource_terraform_dir))
-    os.symlink(deployment_terraform_dir, resource_terraform_dir)
-
     resource_plugins_dir = plugins_dir.replace(
         ctx.target.instance.id, ctx.source.instance.id)
     resource_storage_dir = storage_path.replace(
         ctx.target.instance.id, ctx.source.instance.id)
 
+    if utils.is_using_existing(target=True):
+        # We are going to use a TF binary at another location.
+        # However, we still need to make sure that this directory exists.
+        # Otherwise TF will complain. It does not create it.
+        # In our other scenario, a symlink is created.
+        mkdir_p(resource_terraform_dir)
+    else:
+        # We don't want to put all the plugins for all the node instances in a
+        # deployment multiple times on the system. So here,
+        # we already stored it once on the file system, and now we create
+        # symlinks so other deployments can use it.
+        # TODO: Possibly put this in "apply" and remove the relationship in
+        # the future.
+
+        ctx.logger.info('Creating link {src} {dst}'.format(
+            src=deployment_terraform_dir, dst=resource_terraform_dir))
+        os.symlink(deployment_terraform_dir, resource_terraform_dir)
+
     ctx.logger.info("setting executable_path to {path}".format(
         path=exc_path))
     ctx.logger.info("setting plugins_dir to {dir}".format(

cloudify_tf/terraform/__init__.py

@@ -19,13 +19,7 @@
 
 from contextlib import contextmanager
 
-from cloudify.exceptions import NonRecoverableError
-
-from ..utils import (
-    run_subprocess,
-    get_plugins_dir,
-    get_executable_path,
-    get_resource_config)
+from .. import utils
 
 
 class Terraform(object):
@@ -67,7 +61,7 @@ def set_plugins_dir(path):
         return path
 
     def execute(self, command, return_output=False):
-        return run_subprocess(
+        return utils.run_subprocess(
             command, self.logger, self.root_module,
             self.env, return_output=return_output)
 
@@ -135,14 +129,12 @@ def refresh(self):
 
     @staticmethod
     def from_ctx(ctx, terraform_source):
-        executable_path = get_executable_path()
-        plugins_dir = get_plugins_dir()
-        resource_config = get_resource_config()
-        if not os.path.exists(executable_path):
-            raise NonRecoverableError(
-                "Terraform's executable not found in {0}. Please set the "
-                "'executable_path' property accordingly.".format(
-                    executable_path))
+        executable_path = utils.get_executable_path() or \
+                          utils.get_binary_location_from_rel()
+        plugins_dir = utils.get_plugins_dir()
+        resource_config = utils.get_resource_config()
+        if not os.path.exists(plugins_dir) and utils.is_using_existing():
+            utils.mkdir_p(plugins_dir)
         env_variables = resource_config.get('environment_variables')
         tf = Terraform(
                 ctx.logger,

cloudify_tf/utils.py

@@ -45,6 +45,11 @@
 
 TERRAFORM_STATE_FILE = 'terraform.tfstate'
 
+MASKED_ENV_VARS = {
+    'AWS_ACCESS_KEY_ID',
+    'AWS_SECRET_ACCESS_KEY'
+}
+
 
 def download_file(source, destination):
     run_subprocess(['curl', '-o', source, destination])
@@ -71,12 +76,18 @@ def run_subprocess(command,
         passed_env.update(os.environ)
         passed_env.update(additional_env)
 
+    printed_args = copy.deepcopy(args_to_pass)
+    printed_env = printed_args.get('env', {})
+    for env_var in printed_env.keys():
+        if env_var in MASKED_ENV_VARS:
+            printed_env[env_var] = '****'
+
     logger.info('Running: command={cmd}, '
                 'cwd={cwd}, '
                 'additional_args={args}'.format(
                     cmd=command,
                     cwd=cwd,
-                    args=args_to_pass))
+                    args=printed_args))
 
     process = subprocess.Popen(
         args=command,
@@ -115,6 +126,8 @@ def exclude_file(dirname, filename, excluded_files):
     """
     rel_path = os.path.join(dirname, filename)
     for f in excluded_files:
+        if not f:
+            continue
         if os.path.isfile(f) and rel_path == f:
             return True
     return False
@@ -127,6 +140,8 @@ def exclude_dirs(dirname, subdirs, excluded_files):
     """
     rel_subdirs = [os.path.join(dirname, d) for d in subdirs]
     for f in excluded_files:
+        if not f:
+            continue
         if os.path.isdir(f) and f in rel_subdirs:
             subdirs.remove(ntpath.basename(f))
 
@@ -220,7 +235,14 @@ def _create_source_path(source_tmp_path):
     return source_tmp_path
 
 
-def _unzip_and_set_permissions(zip_file, target_dir):
+def set_permissions(target_file):
+    run_subprocess(
+        ['chmod', 'u+x', target_file],
+        ctx.logger
+    )
+
+
+def unzip_and_set_permissions(zip_file, target_dir):
     """Unzip a file and fix permissions on the files."""
     ctx.logger.debug('Unzipping into {dir}.'.format(dir=target_dir))
 
@@ -238,10 +260,7 @@ def _unzip_and_set_permissions(zip_file, target_dir):
             target_file = os.path.join(target_dir, name)
             ctx.logger.info('Setting executable permission on '
                             '{loc}.'.format(loc=target_file))
-            run_subprocess(
-                ['chmod', 'u+x', target_file],
-                ctx.logger
-            )
+            set_permissions(target_file)
 
 
 def get_instance(_ctx=None, target=False, source=False):
@@ -271,9 +290,63 @@ def get_node(_ctx=None, target=False):
 def is_using_existing(target=True):
     """Decide if we need to do this work or not."""
     resource_config = get_resource_config(target=target)
+    if not target:
+        tf_rel = find_terraform_node_from_rel()
+        if tf_rel:
+            resource_config = tf_rel.target.instance.runtime_properties.get(
+                'resource_config', {})
     return resource_config.get('use_existing_resource', True)
 
 
+def get_binary_location_from_rel():
+    tf_rel = find_terraform_node_from_rel()
+    terraform_config = tf_rel.target.node.properties.get(
+        'terraform_config', {})
+    candidate_a = terraform_config.get('executable_path')
+    candidate_b = get_executable_path()
+    if candidate_b and os.path.isfile(candidate_b):
+        return candidate_b
+    if candidate_a and os.path.isfile(candidate_a):
+        return candidate_a
+    raise NonRecoverableError(
+        "Terraform's executable not found in {0} or {1}. Please set the "
+        "'executable_path' property accordingly.".format(
+            candidate_b, candidate_a))
+
+
+def find_terraform_node_from_rel():
+    return find_rel_by_type(
+        ctx.instance, 'cloudify.terraform.relationships.run_on_host')
+
+
+def find_rel_by_type(node_instance, rel_type):
+    rels = find_rels_by_type(node_instance, rel_type)
+    return rels[0] if len(rels) > 0 else None
+
+
+def find_rels_by_type(node_instance, rel_type):
+    return [x for x in node_instance.relationships
+            if rel_type in x.type_hierarchy]
+
+
+def install_binary(
+        installation_dir,
+        executable_path,
+        installation_source=None):
+
+    if installation_source:
+        installation_zip = os.path.join(installation_dir, 'tf.zip')
+        ctx.logger.info(
+            'Downloading Terraform from {source} into {zip}.'.format(
+                source=installation_source,
+                zip=installation_zip))
+        download_file(installation_zip, installation_source)
+        executable_dir = os.path.dirname(executable_path)
+        unzip_and_set_permissions(installation_zip, executable_dir)
+        os.remove(installation_zip)
+    return executable_path
+
+
 def get_resource_config(target=False):
     """Get the cloudify.nodes.terraform.Module resource_config"""
     instance = get_instance(target=target)
@@ -312,7 +385,7 @@ def update_terraform_source_material(new_source, target=False):
     # Zip the file to store in runtime
     terraform_source_zip = _zip_archive(source_tmp_path)
     base64_rep = _file_to_base64(terraform_source_zip)
-    ctx.logger.warn('The before base64_rep size is {size}.'.format(
+    ctx.logger.info('The before base64_rep size is {size}.'.format(
         size=len(base64_rep)))
 
     instance.runtime_properties['terraform_source'] = base64_rep
@@ -364,6 +437,11 @@ def get_executable_path(target=False):
     if not executable_path:
         executable_path = \
             os.path.join(get_node_instance_dir(target=target), 'terraform')
+    if not os.path.exists(executable_path) and \
+            is_using_existing(target=target):
+        node = get_node(target=target)
+        terraform_config = node.properties.get('terraform_config', {})
+        executable_path = terraform_config.get('executable_path')
     instance.runtime_properties['executable_path'] = executable_path
     ctx.logger.debug('Value executable_path is {loc}.'.format(
         loc=executable_path))
@@ -447,6 +525,7 @@ def remove_dir(folder, desc=''):
         ctx.logger.info('Removing {desc}: {dir}'.format(desc=desc, dir=folder))
         shutil.rmtree(folder)
     elif os.path.islink(folder):
+        ctx.logger.info('Unlinking: {}'.format(folder))
         os.unlink(folder)
     else:
         ctx.logger.info(
@@ -483,7 +562,7 @@ def handle_plugins(plugins, plugins_dir, installation_dir):
             download_file(plugin_zip.name, plugin_url)
             unzip_path = os.path.join(plugins_dir, plugin_name)
             mkdir_p(os.path.basename(unzip_path))
-            _unzip_and_set_permissions(plugin_zip.name, unzip_path)
+            unzip_and_set_permissions(plugin_zip.name, unzip_path)
             os.remove(plugin_zip.name)
 
 

plugin.yaml

@@ -2,7 +2,7 @@ plugins:
   tf:
     executor: central_deployment_agent
     package_name: cloudify-terraform-plugin
-    package_version: '0.14.3'
+    package_version: '0.14.4'
 
 dsl_definitions:
 
@@ -120,7 +120,7 @@ node_types:
       cloudify.interfaces.lifecycle:
         start:
           implementation: tf.cloudify_tf.tasks.apply
-        delete:
+        stop:
           implementation: tf.cloudify_tf.tasks.destroy
       terraform:
         reload: