Add helmfile/terraform runner with hierarchical configuration support (…

…#44)

* Add helmfile/terraform runner with hiera-like configuration support

Signed-off-by: Constantin Muraru <[email protected]>

* Fix tests

Signed-off-by: Constantin Muraru <[email protected]>

* Work1

* Integrate with existing terraform

Signed-off-by: Constantin Muraru <[email protected]>

* Rename ee to hierarchical

* Minor tweaks

* Update requirements.txt

* Add example

* Fix ansible warning

* Fix build

Signed-off-by: cmuraru <[email protected]>

* Tweaks

Signed-off-by: Constantin Muraru <[email protected]>

* Tweaks

* Fixes

* Fixes

* Fixes

* Update readme

* Downgrade aws-cli until it works in Spinnaker

Signed-off-by: cmuraru <[email protected]>

* Add epilog

Signed-off-by: cmuraru <[email protected]>
Signed-off-by: Constantin Muraru <[email protected]>

.gitignore

@@ -1,4 +1,8 @@
 ops.egg-info/
+*.plan
+*.tf.json
+*.tfvars.json
+*.tfstate
 .cache/
 *.pyc
 .terraform

README.md

@@ -59,6 +59,10 @@ ops clusters/mycluster.yaml terraform --path-name aws-eks apply
 
 ![ops-terraform](https://user-images.githubusercontent.com/952836/52021396-9bc1b580-24fd-11e9-9da8-00fb68bd5c72.png)
 
+## Run terraform by using hierarchical configs
+
+See [examples/features/terraform-hierarchical](https://github.com/adobe/ops-cli/tree/master/examples/features/terraform-hierarchical)
+
 ## Create Kubernetes cluster (using AWS EKS)
 
 See [examples/aws-kubernetes](https://github.com/adobe/ops-cli/tree/master/examples/aws-kubernetes)
@@ -85,8 +89,8 @@ pip2 install -U virtualenv
 virtualenv ops
 source ops/bin/activate
 
-# install opswrapper v0.36 stable release
-pip2 install --upgrade https://github.com/adobe/ops-cli/releases/download/0.36/ops-0.36.tar.gz
+# install opswrapper v1.0 stable release
+pip2 install --upgrade https://github.com/adobe/ops-cli/releases/download/1.0/ops-1.0.tar.gz
 
 # Optionally, install terraform to be able to access terraform plugin
 # See https://www.terraform.io/intro/getting-started/install.html
@@ -99,7 +103,7 @@ You can try out `ops-cli`, by using docker. The docker image has all required pr
 
 To start out a container, running the latest `ops-cli` docker image run:
 ```sh
-docker run -it adobe/ops-cli:0.36 bash
+docker run -it adobe/ops-cli:1.0 bash
 ```
 
 After the container has started, you can start using `ops-cli`:

build_scripts/Dockerfile

@@ -73,6 +73,7 @@ RUN curl -sSL https://github.com/databus23/helm-diff/releases/download/v${HELM_D
 
 USER root
 RUN HELM_HOME=/home/ops/.helm helm plugin install https://github.com/futuresimple/helm-secrets
+RUN HELM_HOME=/home/ops/.helm helm plugin install https://github.com/rimusz/helm-tiller
 RUN chown -R ops:ops /home/ops/.helm/plugins
 
 COPY --from=compile-image /azure-cli /home/ops/.local/azure-cli

build_scripts/docker_push.sh

@@ -2,5 +2,5 @@
 set -e
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker tag ops adobe/ops-cli:0.36
-docker push adobe/ops-cli:0.36
+docker tag ops adobe/ops-cli:1.0
+docker push adobe/ops-cli:1.0

examples/features/terraform-hierarchical/.opsconfig.yaml

@@ -0,0 +1,9 @@
+---
+compositions_order:
+  terraform:
+    - account
+    - network
+    - cluster
+    - spinnaker
+  helmfile:
+    - helmfiles

examples/features/terraform-hierarchical/README.md

@@ -0,0 +1,15 @@
+1. Run 'terraform plan' for all compositions for a given cluster:
+```sh
+# generates config and runs terraform
+ops config/env=dev/cluster=cluster1 terraform plan
+```
+
+2. Run 'terraform apply' for all compositions for a given cluster:
+```sh
+ops config/env=dev/cluster=cluster1 terraform apply --skip-plan
+```
+
+3. Run a single composition:
+```sh
+ops config/env=dev/cluster=cluster1/composition=network terraform apply --skip-plan
+```

examples/features/terraform-hierarchical/compositions/terraform/cluster/main.tf

@@ -0,0 +1,10 @@
+variable "config" {}
+
+module "cluster" {
+  source = "../../../modules/cluster"
+  config = var.config
+}
+
+output "cluster_name" {
+  value = var.config.cluster.name
+}

examples/features/terraform-hierarchical/compositions/terraform/network/main.tf

@@ -0,0 +1,6 @@
+variable "config" {}
+
+module "network" {
+  source = "../../../modules/network"
+  config = var.config
+}

examples/features/terraform-hierarchical/config/env=dev/cluster=cluster1/conf.yaml

@@ -0,0 +1,2 @@
+cluster:
+  name: cluster1

examples/features/terraform-hierarchical/config/env=dev/cluster=cluster2/conf.yaml

@@ -0,0 +1,2 @@
+cluster:
+  name: cluster2

examples/features/terraform-hierarchical/config/env=dev/default.yaml

@@ -0,0 +1,18 @@
+account:
+  cloud_provider:
+    aws:
+      profile: test_profile
+
+env:
+  name: dev
+
+region:
+  location: us-east-1
+  name: va6
+
+project:
+  prefix: ee
+
+# This value will be overridden
+cluster:
+  name: default

examples/features/terraform-hierarchical/modules/cluster/main.tf

@@ -0,0 +1,5 @@
+variable "config" {}
+
+output "cluster_name" {
+  value = var.config.cluster.name
+}

examples/features/terraform-hierarchical/modules/network/main.tf

@@ -0,0 +1,17 @@
+variable "config" {}
+
+locals {
+  env     = var.config["env"]
+  region  = var.config["region"]["location"]
+  project = var.config["project"]["prefix"]
+}
+
+#resource "aws_s3_bucket" "bucket" {
+#  bucket = "${local.env}-${local.region}-${local.project}-test-bucket"
+#  acl    = "private"
+
+#  tags = {
+#    Name        = "My bucket"
+#    Environment = "na"
+#  }
+#}

requirements.txt

@@ -1,9 +1,8 @@
 simpledi>=0.2
-awscli==1.16.206
-ansible==2.7.10
-boto3==1.9.196
+awscli==1.16.170
+boto3==1.9.110
 boto==2.49.0
-botocore==1.12.196
+ansible==2.7.12
 PyYAML==3.13
 azure-common==1.1.20
 azure==4.0.0
@@ -15,3 +14,6 @@ hvac==0.9.3
 passgen
 inflection==0.3.1
 kubernetes==9.0.0
+deepmerge==0.0.5
+lru_cache==0.2.3
+backports.functools_lru_cache==1.5

setup.py

@@ -16,7 +16,7 @@
 _requires = [ r for r in open(os.path.sep.join((_mydir,'requirements.txt')), "r").read().split('\n') if len(r)>1 ]
 setup(
     name='ops',
-    version='0.36',
+    version='1.0',
     description='Ops simple wrapper',
     author='Adobe',
     author_email='[email protected]',

src/ops/__init__.py

@@ -58,3 +58,4 @@ def shadow_credentials(self, cmd):
 
 class OpsException(Exception):
     pass
+

src/ops/cli/config.py

@@ -52,6 +52,9 @@ def all(self):
     def __contains__(self, item):
         return item in self.conf or item in self.ops_config
 
+    def __setitem__(self, key, val):
+        self.conf[key] = val
+
     def __getitem__(self, item):
         if item not in self.conf and item not in self.ops_config:
             msg = "Configuration value %s not found; update your %s" % (item, self.cluster_config_path)
@@ -116,6 +119,9 @@ def __init__(self, console_args, cluster_config_path, template):
         self.console_args = console_args
 
     def get(self):
+        if os.path.isdir(self.cluster_config_path):
+            return {"cluster": None, "inventory": None}
+
         data_loader = DataLoader()
         # data_loader.set_vault_password('627VR8*;YU99B')
         variable_manager = VariableManager(loader=data_loader)

src/ops/cli/config_generator.py

@@ -0,0 +1,68 @@
+#Copyright 2019 Adobe. All rights reserved.
+#This file is licensed to you under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License. You may obtain a copy
+#of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software distributed under
+#the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+#OF ANY KIND, either express or implied. See the License for the specific language
+#governing permissions and limitations under the License.
+
+import os
+import logging
+from ops.hierarchical.config_generator import ConfigProcessor
+from ops.cli.parser import SubParserConfig
+
+
+class ConfigGeneratorParserConfig(SubParserConfig):
+    def get_name(self):
+        return 'config'
+
+    def get_help(self):
+        return 'Wrap common terraform tasks with full templated configuration support'
+
+    def configure(self, parser):
+        parser.add_argument('--cwd', dest='cwd', type=str, default="",
+                            help='the working directory')
+        parser.add_argument('--print-data', action='store_true',
+                            help='print generated data on screen')
+        parser.add_argument('--enclosing-key', dest='enclosing_key', type=str,
+                            help='enclosing key of the generated data')
+        parser.add_argument('--output-file', dest='output_file', type=str,
+                            help='output file location')
+        parser.add_argument('--format', dest='output_format', type=str, default="yaml",
+                            help='output file format')
+        parser.add_argument('--filter', dest='filter', action='append',
+                            help='keep these keys from the generated data')
+        parser.add_argument('--exclude', dest='exclude', action='append',
+                            help='exclude these keys from generated data')
+        parser.add_argument('--skip-interpolation-validation', action='store_true',
+                            help='will not throw an error if interpolations can not be resolved')
+        parser.add_argument('--skip-interpolation-resolving', action='store_true',
+                            help='do not perform any AWS calls to resolve interpolations')
+        return parser
+
+    def get_epilog(self):
+        return '''
+        
+        '''
+
+
+class ConfigGeneratorRunner(object):
+    def __init__(self, cluster_config_path):
+        self.cluster_config_path = cluster_config_path
+
+    def run(self, args):
+        logging.basicConfig(level=logging.INFO)
+        args.path = self.cluster_config_path
+        if args.output_file is None:
+            args.print_data = True
+        cwd = args.cwd if args.cwd else os.getcwd()
+        filters = args.filter if args.filter else ()
+        excluded_keys = args.exclude if args.exclude else ()
+
+        generator = ConfigProcessor()
+        generator.process(cwd, args.path, filters, excluded_keys, args.enclosing_key, args.output_format,
+                          args.print_data,
+                          args.output_file, args.skip_interpolation_resolving, args.skip_interpolation_validation,
+                          display_command=False)

src/ops/cli/helmfile.py

@@ -0,0 +1,75 @@
+#Copyright 2019 Adobe. All rights reserved.
+#This file is licensed to you under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License. You may obtain a copy
+#of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software distributed under
+#the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+#OF ANY KIND, either express or implied. See the License for the specific language
+#governing permissions and limitations under the License.
+
+
+import os
+import logging
+from ops.cli.parser import SubParserConfig
+from ops.hierarchical.composition_config_generator import CompositionConfigGenerator
+
+logger = logging.getLogger(__name__)
+
+
+class HelmfileParserConfig(SubParserConfig):
+    def get_name(self):
+        return 'helmfile'
+
+    def get_help(self):
+        return 'Wrap common helmfile tasks using hierarchical configuration support'
+
+    def configure(self, parser):
+        parser.add_argument('subcommand', help='plan | sync | apply | template', type=str)
+        parser.add_argument('extra_args', type=str, nargs='*', help='Extra args')
+        parser.add_argument('--helmfile-path', type=str, default=None, help='Dir to where helmfile.yaml is located')
+        return parser
+
+    def get_epilog(self):
+        return '''
+        Examples:
+            # Run helmfile sync
+            ops data/env=dev/region=va6/project=ee/cluster=experiments/composition=helmfiles helmfile sync
+            # Run helmfile sync for a single chart
+            ops data/env=dev/region=va6/project=ee/cluster=experiments/composition=helmfiles helmfile sync -- --selector chart=nginx-controller
+        '''
+
+
+class HelmfileRunner(CompositionConfigGenerator, object):
+    def __init__(self, ops_config, cluster_config_path):
+        super(HelmfileRunner, self).__init__(["helmfiles"])
+        logging.basicConfig(level=logging.INFO)
+        self.ops_config = ops_config
+        self.cluster_config_path = cluster_config_path
+
+    def run(self, args):
+        config_path_prefix = os.path.join(self.cluster_config_path, '')
+        args.helmfile_path = '../ee-k8s-infra/compositions/helmfiles' if args.helmfile_path is None else os.path.join(args.helmfile_path, '')
+
+        compositions= self.get_sorted_compositions(config_path_prefix)
+        if len(compositions) == 0 or compositions[0] != "helmfiles":
+            raise Exception("Please provide the full path to composition=helmfiles")
+        composition = compositions[0]
+        conf_path = self.get_config_path_for_composition(config_path_prefix, composition)
+        self.generate_helmfile_config(conf_path, args)
+
+        command = self.get_helmfile_command(args)
+        return dict(command=command)
+
+    def generate_helmfile_config(self, path, args):
+        output_file = args.helmfile_path + "/hiera-generated.yaml"
+        logger.info('Generating helmfiles config %s', output_file)
+        self.generator.process(path=path,
+                               filters=["helm"],
+                               output_format="yaml",
+                               output_file=output_file,
+                               print_data=True)
+
+    def get_helmfile_command(self, args):
+        cmd = ' '.join(args.extra_args + [args.subcommand])
+        return "cd {} && helmfile {}".format(args.helmfile_path, cmd)