MVP: Added End to End testing of Reckoner (#88)

* MVP: Added End to End testing of Reckoner

- Added a full end-to-end test with installing charts with reckoner on python 2 & 3
- Added a basic yaml course to test with

.circleci/config.yml

@@ -15,6 +15,100 @@
 version: 2
 
 jobs:
+  end-to-end-2.7:
+    docker:
+      # NOTE: This image isn't actually used since we're mostly working in docker
+      - image: circleci/python:2.7
+    working_directory: ~/reckoner
+    resource_class: small
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run:
+          name: Setup Kube Cluster
+          command: |
+            wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64
+            mv kind-linux-amd64 kind
+            chmod 0755 kind
+
+            ./kind create cluster --wait=90s
+
+            docker run -d --name kube-access --network container:kind-control-plane gcr.io/kubernetes-charts-ci/test-image:v3.2.0 tail -f /dev/null
+            docker exec kube-access mkdir /root/.kube /app/
+            docker cp $(./kind get kubeconfig-path --name=kind) kube-access:/root/.kube/config
+            docker exec kube-access sed -i 's/localhost:.*/localhost:6443/' /root/.kube/config
+            docker exec -it kube-access kubectl version
+
+            docker exec kube-access apk update
+            docker exec kube-access apk add jq
+            docker cp ~/reckoner/ kube-access:/app/.
+            docker exec kube-access pip install /app/reckoner/
+            docker exec kube-access pip list
+
+            docker exec kube-access kubectl -n kube-system create serviceaccount tiller
+            docker exec kube-access kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount kube-system:tiller
+            docker exec kube-access helm init --service-account tiller --wait
+      - run:
+          name: Run Tests
+          command: |
+            docker exec kube-access reckoner --version
+            docker exec -e test_environ_var=testing kube-access reckoner plot /app/reckoner/end_to_end_testing/basic.yml
+            docker exec kube-access sh -c "helm list --output json | jq . -C"
+
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"nginx-ingress\")|.Namespace == \"infra\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"go-harbor\")|.Namespace == \"test\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"redis-env\")|.Namespace == \"redis-test-namespace\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"nginx-ingress\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"go-harbor\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"redis-env\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '[.Releases[].Name] | contains([\"redis-hook\"])|not'"
+  end-to-end-3.7:
+    docker:
+      # NOTE: This image isn't actually used since we're mostly working in docker
+      - image: circleci/python:3.7
+    working_directory: ~/reckoner
+    resource_class: small
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run:
+          name: Setup Kube Cluster
+          command: |
+            wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64
+            mv kind-linux-amd64 kind
+            chmod 0755 kind
+
+            ./kind create cluster --wait=90s
+
+            docker run -d --name kube-access --network container:kind-control-plane gcr.io/kubernetes-charts-ci/test-image:v3.2.0 tail -f /dev/null
+            docker exec kube-access mkdir /root/.kube /app/
+            docker cp $(./kind get kubeconfig-path --name=kind) kube-access:/root/.kube/config
+            docker exec kube-access sed -i 's/localhost:.*/localhost:6443/' /root/.kube/config
+            docker exec -it kube-access kubectl version
+
+            docker exec kube-access apk update
+            docker exec kube-access apk add python3 jq
+            docker cp ~/reckoner/ kube-access:/app/.
+            docker exec kube-access pip3 install /app/reckoner/
+            docker exec kube-access pip3 list
+
+            docker exec kube-access kubectl -n kube-system create serviceaccount tiller
+            docker exec kube-access kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount kube-system:tiller
+            docker exec kube-access helm init --service-account tiller --wait
+      - run:
+          name: Run Tests
+          command: |
+            docker exec kube-access reckoner --version
+            docker exec -e test_environ_var=testing kube-access reckoner plot /app/reckoner/end_to_end_testing/basic.yml
+            docker exec kube-access sh -c "helm list --output json | jq . -C"
+
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"nginx-ingress\")|.Namespace == \"infra\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"go-harbor\")|.Namespace == \"test\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"redis-env\")|.Namespace == \"redis-test-namespace\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"nginx-ingress\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"go-harbor\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '.Releases[]|select(.Name == \"redis-env\")|.Status == \"DEPLOYED\"'"
+            docker exec kube-access sh -c "helm list --output json | jq -e '[.Releases[].Name] | contains([\"redis-hook\"])|not'"
   build-2.7:
     docker:
       - image: circleci/python:2.7
@@ -108,11 +202,26 @@ workflows:
               only: /.*/
             branches:
               only: /.*/
-      - release:
+      - end-to-end-2.7:
           requires:
             - build-2.7
-            - build-3.6
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: /.*/
+      - end-to-end-3.7:
+          requires:
             - build-3.7
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: /.*/
+      - release:
+          requires:
+            - end-to-end-2.7
+            - end-to-end-3.7
           filters:
             tags:
               only: /.*/

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changes
 - Adjusted versioning to be based on git tag
 - Added support for python 3.6 & 3.7 with CI checks
+- Added end-to-end basic tests for CI (invocation regression testing)
 
 ## [1.1.0]
 

end_to_end_testing/basic.yml

@@ -0,0 +1,35 @@
+namespace: test #namespace to install the chart in, defaults to 'kube-system'
+repositories:
+  test_repo:
+    url: https://kubernetes-charts.storage.googleapis.com
+  incubator:
+    url: https://kubernetes-charts-incubator.storage.googleapis.com
+  reactiveops-stable:
+    url: https://charts.reactiveops.com/stable
+  reactiveops-incubator:
+    url: https://charts.reactiveops.com/incubator
+minimum_versions: #set minimum version requirements here
+  helm: 0.0.0
+  reckoner: 0.0.0
+charts:
+  nginx-ingress:
+    repository: stable
+    namespace: infra
+  go-harbor:
+    repository:
+      git: https://github.com/goharbor/harbor-helm.git
+    version: master
+  redis-env:
+    chart: redis
+    namespace: redis-test-namespace
+    repository: stable
+    values:
+      test_environ_var: ${test_environ_var}
+  redis-hook:
+    chart: redis
+    namespace: redis-test-namespace
+    repository: stable
+    hooks:
+      pre_install:
+        - whoami
+      post_install: nonexistant command here

setup.py

@@ -40,7 +40,8 @@
           "GitPython==2.1.3",
           "oyaml>=0.8",
           "coloredlogs==9.0",
-          "semver==2.8.0"
+          "semver==2.8.0",
+          "PyYAML>=5.1",
       ],
       entry_points=''' #for click integration
           [console_scripts]