Molecule tests (dupe to test travis)

.travis.yml

@@ -0,0 +1,18 @@
+---
+sudo: required
+language: python
+services:
+  - docker
+
+# Limit docker version as workaround for: https://github.com/ansible/ansible/issues/35612
+install:
+  - pip install 'docker<3.0'
+  - pip install ansible==2.4.3
+  - pip install molecule
+
+script:
+  - molecule --version
+  - ansible --version
+  - molecule test
+notifications:
+webhooks: https://galaxy.ansible.com/api/v1/notifications/

molecule/default/Dockerfile.j2

@@ -0,0 +1,13 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; fi

molecule/default/INSTALL.rst

@@ -0,0 +1,16 @@
+*******
+Install
+*******
+
+Requirements
+============
+
+* Docker Engine
+* docker-py
+
+Install
+=======
+
+.. code-block:: bash
+
+  $ sudo pip install docker-py

molecule/default/create.yml

@@ -0,0 +1,69 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not lookup('env', 'MOLECULE_DEBUG') | bool }}"
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+    molecule_scenario_directory: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+  tasks:
+    - name: Create Dockerfiles from image names
+      template:
+        src: "{{ molecule_scenario_directory }}/Dockerfile.j2"
+        dest: "{{ molecule_ephemeral_directory }}/Dockerfile_{{ item.image | regex_replace('[^a-zA-Z0-9_]', '_') }}"
+      with_items: "{{ molecule_yml.platforms }}"
+      register: platforms
+
+    - name: Discover local Docker images
+      docker_image_facts:
+        name: "molecule_local/{{ item.item.name }}"
+      with_items: "{{ platforms.results }}"
+      register: docker_images
+
+    - name: Build an Ansible compatible image
+      docker_image:
+        path: "{{ molecule_ephemeral_directory }}"
+        name: "molecule_local/{{ item.item.image }}"
+        dockerfile: "{{ item.item.dockerfile | default(item.invocation.module_args.dest) }}"
+        force: "{{ item.item.force | default(true) }}"
+      with_items: "{{ platforms.results }}"
+      when: platforms.changed or docker_images.results | map(attribute='images') | select('equalto', []) | list | count >= 0
+
+    - name: Create docker network(s)
+      docker_network:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks }}"
+
+    - name: Create molecule instance(s)
+      docker_container:
+        name: "{{ item.name }}"
+        hostname: "{{ item.name }}"
+        image: "molecule_local/{{ item.image }}"
+        state: started
+        recreate: false
+        log_driver: json-file
+        command: "{{ item.command | default('bash -c \"while true; do sleep 10000; done\"') }}"
+        privileged: "{{ item.privileged | default(omit) }}"
+        volumes: "{{ item.volumes | default(omit) }}"
+        capabilities: "{{ item.capabilities | default(omit) }}"
+        exposed_ports: "{{ item.exposed_ports | default(omit) }}"
+        published_ports: "{{ item.published_ports | default(omit) }}"
+        ulimits: "{{ item.ulimits | default(omit) }}"
+        networks: "{{ item.networks | default(omit) }}"
+        dns_servers: "{{ item.dns_servers | default(omit) }}"
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: docker_jobs
+      until: docker_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"

molecule/default/destroy.yml

@@ -0,0 +1,33 @@
+---
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not lookup('env', 'MOLECULE_DEBUG') | bool }}"
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      docker_container:
+        name: "{{ item.name }}"
+        state: absent
+        force_kill: "{{ item.force_kill | default(true) }}"
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: docker_jobs
+      until: docker_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"
+
+    - name: Delete docker network(s)
+      docker_network:
+        name: "{{ item }}"
+        state: absent
+      with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks }}"

molecule/default/molecule.yml

@@ -0,0 +1,56 @@
+---
+dependency:
+  name: galaxy
+  options:
+    role-file: requirements.yml
+    ignore-certs: True
+#    ignore-errors: True
+
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: centos7001
+    image: centos:7
+  - name: xenial001
+    image: solita/ubuntu-systemd:16.04
+    command: /sbin/init
+    privileged: True
+    # groups only required once we set up a mini cluster with spark services
+    #groups:
+    #  - spark_master
+
+provisioner:
+  name: ansible
+  inventory:
+    group_vars:
+      all:
+        #TODO fix issues when trying to deploy the JDK using role(dependency) azavea.java
+        #spark_install_java: yes
+        spark_install_java: no
+        spark_version: "2.1.1-bin-hadoop2.7"
+        # following is set to ensure SPARK_LOCAL_DIRS is set in spark-env.sh
+        spark_local_dirs: ["/tmp"]
+        # Test spark-services later
+        #spark_services_install_enabled: False
+  lint:
+    name: ansible-lint
+
+scenario:
+  name: default
+  # Disabled some of the stages (for ex. 'lint' would throw many errors)
+  test_sequence:
+    - destroy
+    - dependency
+    - syntax
+#    - lint
+    - create
+    - converge
+    - idempotence
+    - verify
+
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

molecule/default/playbook.yml

@@ -0,0 +1,6 @@
+---
+- name: Converge
+  hosts: all
+  roles:
+    - role: geerlingguy.java
+    - role: ansible-spark

molecule/default/prepare.yml

@@ -0,0 +1,5 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks: []

molecule/default/tests/test_default.py

@@ -0,0 +1,69 @@
+import os
+import re
+import pytest
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_hosts_file(host):
+    f = host.file('/etc/hosts')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.group == 'root'
+
+
+@pytest.mark.common
+def test_spark_user_dir_exists(host):
+    f = host.file('/var/lib/spark/')
+    assert f.exists
+
+
+@pytest.mark.common
+def test_spark_defaults_conf(host):
+    f = host.file('/usr/lib/spark/conf/spark-defaults.conf')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.mode == 0o644
+    assert f.contains('spark.master')
+
+
+@pytest.mark.common
+def test_spark_env_sh(host):
+    f = host.file('/usr/lib/spark/conf/spark-env.sh')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.mode == 0o644
+    assert f.contains('LD_LIBRARY_PATH=')
+    assert f.contains('SPARK_LOG_DIR=')
+    assert f.contains('SPARK_PID_DIR=')
+    # following env var is set only if 'spark_local_dirs' is configured
+    assert f.contains('SPARK_LOCAL_DIRS=')
+    assert f.contains('SPARK_LOCAL_DIRS="${SPARK_LOCAL_DIRS:-/tmp}"')
+
+
+@pytest.mark.common
+def test_spark_log4j_properties(host):
+    f = host.file('/usr/lib/spark/conf/log4j.properties')
+
+    assert f.exists
+    assert f.mode == 0o644
+    assert f.contains('log4j.rootCategory=')
+    assert f.contains('log4j.appender.console=')
+    assert f.contains('log4j.logger.org.apache.spark.repl.Main')
+
+
+@pytest.mark.common
+def test_spark_running_SparkPi(host):
+    sparkTasks = 100
+    sparkParams = ''
+    sparkCmd = 'cd /usr/lib/spark/bin ; ./run-example {} SparkPi {}'.format(sparkParams, sparkTasks)
+    out = host.check_output(sparkCmd)
+
+    assert re.match("^Pi is roughly 3.14[0-9]+", out)
+    # a successful job will return a rough estimation of Pi like
+    #Pi is roughly 3.1412735141273

requirements.yml

@@ -0,0 +1,7 @@
+---
+- name: azavea.java
+  version: 0.6.1
+
+## More standard role to deploy Open JDK, tested on major distros: debian/ubuntu, centos/redhat
+- name: geerlingguy.java
+  version: 1.7.6