common.conf

# Director 2.3 Config file
# Creates a small cluster including a Cloudera Data Science Workbench gateway node
#
#
# Copyright (c) 2016 Cloudera, Inc. All rights reserved.
#

#
# Simple AWS Cloudera Director configuration file with automatic role assignments
# that works as expected if you use a single instance type for all cluster nodes
#

#
# Cluster name
#
name: cdh-5-14

#
# Cloud provider configuration (credentials, region or zone and optional default image)
#

ssh {

    username: ${SSH_USERNAME}
    privateKey: ${SSH_PRIVATEKEY}
}


#
# Configuration for Cloudera Manager. Cloudera Director can use an existing instance
# or bootstrap everything from scratch for a new cluster
#

cloudera-manager {

    instance: ${instances.master} {
    instanceNamePrefix: cm-${name}
        tags {
            application: "Cloudera Manager 5"
        }
    
   }
    krbAdminUsername: ${?krbAdminUsername}
    krbAdminPassword: ${?krbAdminPassword}

   configs {
        # CLOUDERA_MANAGER corresponds to the Cloudera Manager Server configuration options

   	 CLOUDERA_MANAGER {
	    enable_api_debug: false
            custom_banner_html: "Managed by Cloudera Director"
            MANAGES_PARCELS: true
	    enable_faster_bootstrap: true
            KDC_TYPE: ${?KDC_TYPE}
	    AD_KDC_DOMAIN: ${?AD_KDC_DOMAIN}
	    KDC_HOST: ${?KDC_HOST_IP} # Using IP because of DSE-1796
            SECURITY_REALM: ${?SECURITY_REALM}
            KRB_MANAGE_KRB5_CONF: ${?KRB_MANAGE_KRB5_CONF}
	    # Note use of aes256 - need to ensure ALL cluster members have the necessary JCE policy files.
	    # If you find that you can get a ticket but not use it then this is likely the problem!
            KRB_ENC_TYPES: ${?KRB_ENC_TYPES}
   	}
    }
    csds: [
        "https://archive.cloudera.com/cdsw/1/csd/CLOUDERA_DATA_SCIENCE_WORKBENCH-1.3.0.jar",
        "http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.2.0.cloudera2.jar"
    ]

    #
    # Automatically activate 60-Day Cloudera Enterprise Trial
    #
    enableEnterpriseTrial: true

    ## Dont install java but let the bootstrap scripts do the job
    javaInstallationStrategy: NONE

    repository: "http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14/"
    repositoryKeyUrl: "http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera"
}

#
# Cluster description
#

cluster {

    products {
      CDH: 5.14 # includes Hive and Spark
      SPARK2: 2.2.0.cloudera2 
      Anaconda: 4.4.0
      CDSW: 1.3.0
    }


    parcelRepositories:
    ["http://archive.cloudera.com/cdh5/parcels/5.14/", 
    "https://archive.cloudera.com/spark2/parcels/2.2.0.cloudera2/",
    "https://repo.continuum.io/pkgs/misc/parcels/"
    "https://archive.cloudera.com/cdsw/1/parcels/1.3.0/"
    ]

    services: [HDFS, YARN, CDSW, SPARK2_ON_YARN, HIVE, OOZIE, IMPALA, HUE]

    configs {
        CDSW {
            "cdsw.domain.config": cdsw.placeholder-domain.com # The fully qualified domain name for the CDSW host
        }
    }

    masters {
      count: 1
      instance: ${instances.master}

      roles {
        HDFS: [NAMENODE, SECONDARYNAMENODE]
        YARN: [RESOURCEMANAGER, JOBHISTORY]
 	SPARK2_ON_YARN: [SPARK2_YARN_HISTORY_SERVER]
        HIVE: [HIVESERVER2, HIVEMETASTORE]
        OOZIE: [OOZIE_SERVER]
	IMPALA: [CATALOGSERVER, STATESTORE]
        HUE: [HUE_SERVER, KT_RENEWER]
      }
       YARN {
             RESOURCEMANAGER {
	                   yarn_scheduler_maximum_allocation_mb: 8192
             		   yarn_scheduler_maximum_allocation_vcores: 4
             }
       }

    }

    workers {
      count: ${CDHWORKER_NODECOUNT}

      instance: ${instances.worker}

      roles {
        HDFS: [DATANODE]
	IMPALA: [IMPALAD]
        YARN: [NODEMANAGER]
      }
      configs {
      	 YARN {
	      	    NODEMANAGER {
		   	          yarn_nodemanager_resource_memory_mb: ${YARN_RAM}
			            yarn_nodemanager_resource_cpu_vcores: ${YARN_VCORES}
	            }
	       }
	    }
    }
    cdswmasters {
      count: 1

      instance: ${instances.cdswmaster} {
              bootstrapScriptsPaths:${instances.cdswmaster.bootstrapScriptsPaths} ["scripts/cdswmaster-bootstrap-script.sh"]
      } 

      roles {
	  HDFS: [GATEWAY]
	  YARN: [GATEWAY]
	  CDSW: [CDSW_MASTER, CDSW_APPLICATION, CDSW_DOCKER]
	  SPARK2_ON_YARN: [GATEWAY]
	  HIVE: [GATEWAY]
      }
      configs {
            SPARK2_ON_YARN {
                GATEWAY {
                    "spark2-conf/spark-env.sh_client_config_safety_valve": 
                    """if [ -z ${PYSPARK_PYTHON} ]; 
                    then export PYSPARK_PYTHON=/opt/cloudera/parcels/Anaconda/bin/python spark-submit pyspark_script.py; fi"""
                }
            }
            CDSW {
                CDSW_DOCKER {
                    "cdsw.docker.devices.config": ${DOCKER_DISKS} # related to the ebs configuration above
                }
            }
        }
    }
    cdswworkers {
        count: ${CDSWWORKER_NODECOUNT}
        minCount: ${CDSWWORKER_NODECOUNT}
        instance: ${instances.cdswworker} 

        roles {
            HDFS: [GATEWAY]
            YARN: [GATEWAY]
            CDSW: [CDSW_WORKER, CDSW_DOCKER]
            SPARK2_ON_YARN: [GATEWAY]
            HIVE: [GATEWAY]
        }

        configs {
            CDSW {
                CDSW_DOCKER {
                    "cdsw.docker.devices.config": ${DOCKER_DISKS} # related to the ebs configuration above
                }
            }
        }
    }
   instancePostCreateScriptsPaths:  ["scripts/cdsw_postcreate.sh"] 

    administrationSettings {
      # If enabled, Director will attempt to automatically repair
      # clusters whose instances have been terminated in the cloud provider.

      autoRepairEnabled: true
      autoRepairCooldownPeriodInSeconds: 300
    }

}

common-instance-properties.bootstrapScriptsPaths: ["scripts/clean-yum-cache.sh", "scripts/kerberos_client.sh", "scripts/java8-bootstrap-script.sh", "scripts/users.sh"]