-
Notifications
You must be signed in to change notification settings - Fork 0
/
demo.blueprint
80 lines (80 loc) · 15.2 KB
/
demo.blueprint
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
{
"configurations": [
{
"yarn-env": {
"properties": {
"content": "export HADOOP_YARN_HOME={{hadoop_yarn_home}}\r\nexport YARN_LOG_DIR={{yarn_log_dir_prefix}}\/$USER\r\nexport YARN_PID_DIR={{yarn_pid_dir_prefix}}\/$USER\r\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\r\nexport JAVA_HOME={{java64_home}}\r\nexport JAVA_LIBRARY_PATH=\"${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}\"\r\n# We need to add the EWMA appender for the yarn daemons only;\r\n# however, YARN_ROOT_LOGGER is shared by the yarn client and the\r\n# daemons. This is restrict the EWMA appender to daemons only.\r\nINVOKER=\"${0##*\/}\"\r\nif [ \"$INVOKER\" == \"yarn-daemon.sh\" ]; then\r\n export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA}\r\nfi\r\n# User for YARN daemons\r\nexport HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}\r\n# resolve links - $0 may be a softlink\r\nexport YARN_CONF_DIR=\"${YARN_CONF_DIR:-$HADOOP_YARN_HOME\/conf}\"\r\n# some Java parameters\r\n# export JAVA_HOME=\/home\/y\/libexec\/jdk1.6.0\/\r\nif [ \"$JAVA_HOME\" != \"\" ]; then\r\n#echo \"run java in $JAVA_HOME\"\r\nJAVA_HOME=$JAVA_HOME\r\nfi\r\nif [ \"$JAVA_HOME\" = \"\" ]; then\r\necho \"Error: JAVA_HOME is not set.\"\r\nexit 1\r\nfi\r\nJAVA=$JAVA_HOME\/bin\/java\r\nJAVA_HEAP_MAX=-Xmx1000m\r\n# For setting YARN specific HEAP sizes please use this\r\n# Parameter and set appropriately\r\nYARN_HEAPSIZE={{yarn_heapsize}}\r\n# check envvars which might override default args\r\nif [ \"$YARN_HEAPSIZE\" != \"\" ]; then\r\nJAVA_HEAP_MAX=\"-Xmx\"\"$YARN_HEAPSIZE\"\"m\"\r\nfi\r\n# Resource Manager specific parameters\r\n# Specify the max Heapsize for the ResourceManager using a numerical value\r\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\r\n# the value to 1000.\r\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\r\n# and\/or YARN_RESOURCEMANAGER_OPTS.\r\n# If not specified, the default value will be picked from either YARN_HEAPMAX\r\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\r\nexport YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}\r\n# Specify the JVM options to be used when starting the ResourceManager.\r\n# These options will be appended to the options specified as YARN_OPTS\r\n# and therefore may override any similar flags set in YARN_OPTS\r\n#export YARN_RESOURCEMANAGER_OPTS=\r\n# Node Manager specific parameters\r\n# Specify the max Heapsize for the NodeManager using a numerical value\r\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\r\n# the value to 1000.\r\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\r\n# and\/or YARN_NODEMANAGER_OPTS.\r\n# If not specified, the default value will be picked from either YARN_HEAPMAX\r\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\r\nexport YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}\r\n# Specify the max Heapsize for the timeline server using a numerical value\r\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\r\n# the value to 1024.\r\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\r\n# and\/or YARN_TIMELINESERVER_OPTS.\r\n# If not specified, the default value will be picked from either YARN_HEAPMAX\r\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\r\nexport YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}\r\n# Specify the JVM options to be used when starting the NodeManager.\r\n# These options will be appended to the options specified as YARN_OPTS\r\n# and therefore may override any similar flags set in YARN_OPTS\r\n#export YARN_NODEMANAGER_OPTS=\r\n# so that filenames w\/ spaces are handled correctly in loops below\r\nIFS=\r\n# default log directory and file\r\nif [ \"$YARN_LOG_DIR\" = \"\" ]; then\r\nYARN_LOG_DIR=\"$HADOOP_YARN_HOME\/logs\"\r\nfi\r\nif [ \"$YARN_LOGFILE\" = \"\" ]; then\r\nYARN_LOGFILE=\'yarn.log\'\r\nfi\r\n# default policy file for service-level authorization\r\nif [ \"$YARN_POLICYFILE\" = \"\" ]; then\r\nYARN_POLICYFILE=\"hadoop-policy.xml\"\r\nfi\r\n# restore ordinary behaviour\r\nunset IFS\r\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR\"\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR\"\r\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE\"\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE\"\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME\"\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING\"\r\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\r\nexport YARN_NODEMANAGER_OPTS=\"$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT -javaagent:\/opt\/jmx_javaagent.jar=20102:\/etc\/jmx_exporter\/jmx_exporter.yaml\"\r\nexport YARN_RESOURCEMANAGER_OPTS=\"$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT -javaagent:\/opt\/jmx_javaagent.jar=20101:\/etc\/jmx_exporter\/jmx_exporter.yaml\"\r\nif [ \"x$JAVA_LIBRARY_PATH\" != \"x\" ]; then\r\nYARN_OPTS=\"$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH\"\r\nfi\r\nYARN_OPTS=\"$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE\"\r\nYARN_OPTS=\"$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}\""
}
}
},
{
"hadoop-env": {
"properties": {
"content": "# Set Hadoop-specific environment variables here.\r\n# The only required environment variable is JAVA_HOME. All others are\r\n# optional. When running a distributed configuration it is best to\r\n# set JAVA_HOME in this file, so that it is correctly defined on\r\n# remote nodes.\r\n# The java implementation to use. Required.\r\nexport JAVA_HOME={{java_home}}\r\nexport HADOOP_HOME_WARN_SUPPRESS=1\r\n# Hadoop home directory\r\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\r\n# Hadoop Configuration Directory\r\n{# this is different for HDP1 #}\r\n# Path to jsvc required by secure HDP 2.0 datanode\r\nexport JSVC_HOME={{jsvc_path}}\r\n# The maximum amount of heap to use, in MB. Default is 1000.\r\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\r\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\r\n# Extra Java runtime options. Empty by default.\r\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\r\n# Command specific options appended to HADOOP_OPTS when specified\r\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}\/$USER\/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}\/$USER\/gc.log-`date +\'%Y%m%d%H%M\'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\r\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\r\n{% if java_version < 8 %}\r\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}\/$USER\/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}\/$USER\/gc.log-`date +\'%Y%m%d%H%M\'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\r\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"\/usr\/hdp\/current\/hadoop-hdfs-namenode\/bin\/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\r\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=\/var\/log\/hadoop\/$USER\/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:\/var\/log\/hadoop\/$USER\/gc.log-`date +\'%Y%m%d%H%M\'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\r\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"\/usr\/hdp\/current\/hadoop-hdfs-secondarynamenode\/bin\/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\r\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\r\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\r\n{% else %}\r\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}\/$USER\/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}\/$USER\/gc.log-`date +\'%Y%m%d%H%M\'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\r\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"\/usr\/hdp\/current\/hadoop-hdfs-namenode\/bin\/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 -javaagent:\/opt\/jmx_javaagent.jar=20103:\/etc\/jmx_exporter\/jmx_exporter.yaml\"\r\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=\/var\/log\/hadoop\/$USER\/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:\/var\/log\/hadoop\/$USER\/gc.log-`date +\'%Y%m%d%H%M\'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT -javaagent:\/opt\/jmx_javaagent.jar=20104:\/etc\/jmx_exporter\/jmx_exporter.yaml\"\r\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"\/usr\/hdp\/current\/hadoop-hdfs-secondarynamenode\/bin\/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\r\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\r\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\r\n{% endif %}\r\nHADOOP_NFS3_OPTS=\"-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}\"\r\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\r\n# On secure datanodes, user to run the datanode as after dropping privileges\r\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\r\n# Extra ssh options. Empty by default.\r\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\r\n# Where log files are stored. $HADOOP_HOME\/logs by default.\r\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}\/$USER\r\n# History server logs\r\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}\/$USER\r\n# Where log files are stored in the secure data environment.\r\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}\/$HADOOP_SECURE_DN_USER\r\n# File naming remote slave hosts. $HADOOP_HOME\/conf\/slaves by default.\r\n# export HADOOP_SLAVES=${HADOOP_HOME}\/conf\/slaves\r\n# host:path where hadoop code should be rsync\'d from. Unset by default.\r\n# export HADOOP_MASTER=master:\/home\/$USER\/src\/hadoop\r\n# Seconds to sleep between slave commands. Unset by default. This\r\n# can be useful in large clusters, where, e.g., slave rsyncs can\r\n# otherwise arrive faster than the master can service them.\r\n# export HADOOP_SLAVE_SLEEP=0.1\r\n# The directory where pid files are stored. \/tmp by default.\r\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}\/$USER\r\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}\/$HADOOP_SECURE_DN_USER\r\n# History server pid\r\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}\/$USER\r\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\r\n# A string representing this instance of hadoop. $USER by default.\r\nexport HADOOP_IDENT_STRING=$USER\r\n# The scheduling priority for daemon processes. See \'man nice\'.\r\n# export HADOOP_NICENESS=10\r\n# Add database libraries\r\nJAVA_JDBC_LIBS=\"\"\r\nif [ -d \"\/usr\/share\/java\" ]; then\r\n for jarFile in `ls \/usr\/share\/java | grep -E \"(mysql|ojdbc|postgresql|sqljdbc)\" 2>\/dev\/null`\r\n do\r\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\n done\r\nfi\r\n# Add libraries to the hadoop classpath - some may not need a colon as they already include it\r\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}\r\n# Setting path to hdfs command line\r\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\r\n# Mostly required for hadoop 2.0\r\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\r\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\r\n# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. \r\n# Makes sense to fix only when runing DN as root \r\nif [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then\r\n {% if is_datanode_max_locked_memory_set %}\r\n ulimit -l {{datanode_max_locked_memory}}\r\n {% endif %}\r\n ulimit -n {{hdfs_user_nofile_limit}}\r\nfi"
}
}
},
{
"zookeeper-env": {
"properties": {
"content": "export JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\nexport JMXLOCALONLY=true\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Dcom.sun.management.jmxremote.port=20009\"\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}"
}
}
}
],
"host_groups": [
{
"name": "master",
"components": [
{
"name": "NAMENODE"
},
{
"name": "SECONDARY_NAMENODE"
},
{
"name": "RESOURCEMANAGER"
},
{
"name": "APP_TIMELINE_SERVER"
},
{
"name": "HISTORYSERVER"
},
{
"name": "ZOOKEEPER_SERVER"
}
],
"cardinality": "1"
},
{
"name": "slave_1",
"components": [
{
"name": "DATANODE"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "NODEMANAGER"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "ZOOKEEPER_CLIENT"
}
],
"cardinality": "2"
}
],
"Blueprints": {
"blueprint_name": "multi-node-hdfs-yarn",
"stack_name": "HDP",
"stack_version": "2.5"
}
}