diff --git a/dockers/cdh6.3.2/Dockerfile b/dockers/cdh6.3.2/Dockerfile new file mode 100644 index 0000000..3ee7cf0 --- /dev/null +++ b/dockers/cdh6.3.2/Dockerfile @@ -0,0 +1,15 @@ +FROM dss_linkis:v1 + +MAINTAINER Zsy "mr.zshuya@gmail.com" + +RUN pip3 install --no-cache-dir -r /wedatasphere/docker/conf/requirements.txt -i \ + http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + +COPY docker /wedatasphere/docker/ +COPY sbin /wedatasphere/sbin/ + +RUN chown -R hdfs:hdfs /wedatasphere && chmod +x /wedatasphere/docker/script/run.sh + +WORKDIR /wedatasphere + +CMD ["/bin/bash", "/wedatasphere/docker/script/run.sh"] \ No newline at end of file diff --git a/dockers/cdh6.3.2/README.md b/dockers/cdh6.3.2/README.md new file mode 100644 index 0000000..e91f418 --- /dev/null +++ b/dockers/cdh6.3.2/README.md @@ -0,0 +1,23 @@ +WeDataSphere + +需求文档： + +https://github.com/WeBankFinTech/WeDataSphere/issues/39 + +使用流程： + +1、安装部署： + +参考安装部署文档即可快速安装使用WeDataSphere +[安装部署文档](https://github.com/MrZsy/WeDataSphere/blob/master/docs/dockers/cdh6.3.2/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3.md) + +2、组件升级参考： +有需要组件升级及使用其他版本组件的开发同学可以参考[升级指南](https://github.com/MrZsy/WeDataSphere/blob/master/docs/dockers/cdh6.3.2/%E5%8D%87%E7%BA%A7%E6%8C%87%E5%8D%97.md) 快速更改版本。 + + +3、常见问题参考： +[常见问题](https://github.com/MrZsy/WeDataSphere/blob/master/docs/dockers/cdh6.3.2/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98.md) + + +4、其它参考官方文档： +[开发文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%BC%80%E5%8F%91%E6%96%87%E6%A1%A3) \ No newline at end of file diff --git a/dockers/cdh6.3.2/docker/conf/conf.ini b/dockers/cdh6.3.2/docker/conf/conf.ini new file mode 100644 index 0000000..05515a2 --- /dev/null +++ b/dockers/cdh6.3.2/docker/conf/conf.ini @@ -0,0 +1,5 @@ +[hiveMetaData] +hive.meta.host: 172.16.13.224 +hive.meta.db: metastore +hive.meta.user: hive +hive.meta.password: 123456789 diff --git a/dockers/cdh6.3.2/docker/conf/requirements.txt b/dockers/cdh6.3.2/docker/conf/requirements.txt new file mode 100644 index 0000000..5d02b00 --- /dev/null +++ b/dockers/cdh6.3.2/docker/conf/requirements.txt @@ -0,0 +1,3 @@ +PyMySQL==1.0.2 +PyYAML==6.0 +configparser \ No newline at end of file diff --git a/dockers/cdh6.3.2/docker/script/db_update.py b/dockers/cdh6.3.2/docker/script/db_update.py new file mode 100644 index 0000000..b8e4fc7 --- /dev/null +++ b/dockers/cdh6.3.2/docker/script/db_update.py @@ -0,0 +1,125 @@ +# !/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +@IDE : PyCharm +@File : db_update.py +@Time : 2023-03-07 9:51 +@Place : BeiJing +@Author : Zsy +@Version : 1.0 +""" +import os +import pymysql +import parse_yarn_ip + +table_list = [ + "dss_appconn_instance", + "dss_workspace_dictionary", + "dss_workspace_menu_appconn", + "linkis_ps_bml_resources_task", + "linkis_ps_bml_resources_version", + "linkis_ps_dm_datasource_env", + "linkis_cg_rm_external_resource_provider" +] + + +def get_host_ip(): + import socket + # 获取计算机名称 + hostname = socket.gethostname() + # 获取本机IP + # ip = os.getenv("HOST_IP") + ip = socket.gethostbyname(hostname) + return ip + + +def conn_mysql(): + conn = pymysql.connect(host="127.0.0.1", port=3306, user="dss_server", password="dssServer123.", + db="dss_server", cursorclass=pymysql.cursors.DictCursor, charset='utf8') + return conn + + +def update_ip(ip): + db_conn = conn_mysql() + cursor = db_conn.cursor() + old_ip = "172.17.0.5" + for table in table_list: + sql = f"select * from dss_server.{table};" + cursor.execute(sql) + res = cursor.fetchall() + print("更新数据库表: {}".format(table)) + if table == "dss_appconn_instance": + for i in res: + data = dict(i) + url = data.get("url") + home_page_url = data.get("homepage_uri") + if url and old_ip in url: + url = url.replace(old_ip, os.getenv("HOST_IP")) + idx = data.get("id") + update_sql = f"update dss_server.{table} set url = '{url}' where id ={idx};" + cursor.execute(update_sql) + db_conn.commit() + if home_page_url and old_ip in home_page_url: + url = home_page_url.replace(old_ip, os.getenv("HOST_IP")) + idx = data.get("id") + update_sql = f"update dss_server.{table} set homepage_uri = '{url}' where id = {idx};" + cursor.execute(update_sql) + db_conn.commit() + if table == "dss_workspace_dictionary": + for i in res: + data = dict(i) + url = data.get("url") + if url and old_ip in url: + url = url.replace(old_ip, ip) + idx = data.get("id") + update_sql = f"update dss_server.{table} set url = '{url}' where id ={idx};" + cursor.execute(update_sql) + db_conn.commit() + pass + if table == "dss_workspace_menu_appconn": + for i in res: + data = dict(i) + manual_button_url = data.get("manual_button_url") + if manual_button_url and old_ip in manual_button_url: + url = manual_button_url.replace(old_ip, ip) + idx = data.get("id") + update_sql = f"update dss_server.{table} set manual_button_url = '{url}' where id ={idx};" + cursor.execute(update_sql) + db_conn.commit() + pass + if table == "linkis_ps_bml_resources_task" or table == "linkis_ps_bml_resources_version": + update_sql = f"update dss_server.{table} set client_ip = '{ip}';" + cursor.execute(update_sql) + db_conn.commit() + if table == "linkis_ps_dm_datasource_env": + for i in res: + data = dict(i) + idx = data.get("id") + parameter = data.get("parameter") + if parameter and old_ip in parameter: + parameter = parameter.replace(old_ip, ip) + update_sql = f"update dss_server.{table} set parameter = '{parameter}' where id = {idx};" + cursor.execute(update_sql) + db_conn.commit() + if table == "linkis_cg_rm_external_resource_provider": + yarn_ip = parse_yarn_ip.run() + for i in res: + data = dict(i) + idx = data.get("id") + # config = data.get("config").replace("http://172.16.13.131:8088", yarn_ip) + config = data.get("config") + if config and "172.16.13.131" in config: + config = config.replace("http://172.16.13.131:8088", yarn_ip) + update_sql = f"update dss_server.{table} set config = '{config}' where id = {idx};" + cursor.execute(update_sql) + db_conn.commit() + + +def run(): + ip = get_host_ip() + update_ip(ip) + print("数据库更新成功") + + +if __name__ == "__main__": + run() diff --git a/dockers/cdh6.3.2/docker/script/parse_yarn_ip.py b/dockers/cdh6.3.2/docker/script/parse_yarn_ip.py new file mode 100644 index 0000000..33e9c49 --- /dev/null +++ b/dockers/cdh6.3.2/docker/script/parse_yarn_ip.py @@ -0,0 +1,67 @@ +# !/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +@IDE : PyCharm +@File : parse_yarn_ip.py +@Time : 2023-03-06 15:43 +@Place : BeiJing +@Author : Zsy +@Version : 1.0 +""" + +from xml.etree.ElementTree import ElementTree + + +def read_xml(in_path): + """ + 读取并解析xml文件 + in_path: xml路径 + return: ElementTree + """ + tree = ElementTree() + tree.parse(in_path) + return tree + + +def find_nodes(tree, path): + """ + 查找某个路径匹配的所有节点 + tree: xml树 + path: 节点路径 + """ + return tree.findall(path) + + +def get_yarn_ips(nodelist, kv_map): + """ + 获取yarn ip + nodelist: 节点列表 + kv_map: 匹配属性及属性值map + + """ + k_list, ip_list = [], [] + for node in nodelist: + name = node.find('name').text + if name == kv_map.get("name"): + for i in node.find('value').text.split(","): + k_list.append("yarn.resourcemanager.webapp.address." + i) + if not k_list: + k_list.append("yarn.resourcemanager.webapp.address") + for node in nodelist: + name = node.find('name').text + if name in k_list: + ip_list.append(node.find('value').text) + return ip_list + + +def run(): + tree = read_xml("/etc/hadoop/conf/yarn-site.xml") + # tree = read_xml("/wedatasphere/cdh/config/hadoop/yarn-site.xml") + nodes = find_nodes(tree, "property") + ips = get_yarn_ips(nodes, {"name": "yarn.resourcemanager.ha.rm-ids"}) + ips = ["http://" + ip for ip in ips if "http://" not in ip] + return ";".join(ips) + + +if __name__ == "__main__": + run() diff --git a/dockers/cdh6.3.2/docker/script/run.sh b/dockers/cdh6.3.2/docker/script/run.sh new file mode 100644 index 0000000..3e417d4 --- /dev/null +++ b/dockers/cdh6.3.2/docker/script/run.sh @@ -0,0 +1,16 @@ +echo "启动mysql" +/etc/init.d/mysqld start --user=mysql & +sleep 3 +echo "启动nginx" +/usr/sbin/nginx -c /etc/nginx/nginx.conf +echo "配置hdfs路径" +source /etc/profile +echo "${CDH_HOME}" +sh ${CDH_HOME}/bin/hdfs dfs -mkdir -p /tmp/test/linkis +sh ${CDH_HOME}/bin/hdfs dfs -chmod 775 /tmp/test/linkis +echo "初始化配置文件" +python3 /wedatasphere/docker/script/db_update.py +python3 /wedatasphere/docker/script/update_config.py +/usr/sbin/nginx -s reload +echo "启动服务" +su - hdfs -s /bin/bash /wedatasphere/sbin/start-all.sh diff --git a/dockers/cdh6.3.2/docker/script/update_config.py b/dockers/cdh6.3.2/docker/script/update_config.py new file mode 100644 index 0000000..94eb866 --- /dev/null +++ b/dockers/cdh6.3.2/docker/script/update_config.py @@ -0,0 +1,144 @@ +# !/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +@IDE : PyCharm +@File : update_config.py +@Time : 2023-03-06 16:54 +@Place : BeiJing +@Author : Zsy +@Version : 1.0 +""" +import yaml +import configparser +import socket + +file_list = [ + "/wedatasphere/install/dss_linkis/dss/conf/config.sh", + "/wedatasphere/install/dss_linkis/dss/conf/application-dss.yml", + "/wedatasphere/install/dss_linkis/dss/conf/dss.properties", + "/wedatasphere/install/dss_linkis/dss/conf/dss-workflow-server.properties", + "/wedatasphere/install/dss_linkis/linkis/conf/application-linkis.yml", + "/wedatasphere/install/dss_linkis/linkis/conf/application-eureka.yml", + "/wedatasphere/install/dss_linkis/linkis/conf/linkis-env.sh", + "/wedatasphere/install/dss_linkis/linkis/sbin/common.sh", + "/wedatasphere/install/dss_linkis/linkis/conf/linkis.properties", + "/wedatasphere/install/dss_linkis/linkis/conf/linkis-ps-publicservice.properties", + "/wedatasphere/install/dss_linkis/web/config.sh", + "/wedatasphere/install/schedulis/schedulis_0.7.1_exec/plugins/jobtypes/linkis/plugin.properties", + "/wedatasphere/install/schedulis/schedulis_0.7.1_exec/plugins/jobtypes/linkis/bin/config.sh", + "/wedatasphere/install/schedulis/schedulis_0.7.1_exec/conf/host.properties", + "/etc/nginx/conf.d/dss.conf", + "/etc/nginx/conf.d/exchangis.conf", + "/etc/nginx/conf.d/streamis.conf" +] + +sh_file_list, yml_file_list, pro_file_list = [], [], [] + + +def file_types(): + for file_path in file_list: + if file_path.endswith(".sh") or file_path.endswith(".conf"): + sh_file_list.append(file_path) + if file_path.endswith(".yml"): + yml_file_list.append(file_path) + if file_path.endswith(".properties"): + pro_file_list.append(file_path) + + +def get_host_ip(): + # 获取计算机名称 + hostname = socket.gethostname() + # 获取本机IP + ip = socket.gethostbyname(hostname) + # ip = os.getenv("HOST_IP") + return ip + + +# 读文件 +def file_read(read_path): + # 打开文件 + file_data = open(read_path, encoding='utf8') + file_read_data = file_data.read() + # 关闭文件 + file_data.close() + return file_read_data + + +# 写文件 +def file_write(write_path, res): + # 打开文件 + with open(write_path, 'w', encoding='utf8') as file_data: + file_data.write(str(res)) + file_data.close() + + +def read_file_sh(ip): + for read_path in sh_file_list: + if read_path.endswith("common.sh"): + file_data = file_read(read_path) + hostname = socket.gethostname() + res = file_data.replace("dss.node.cn", hostname) + file_write(read_path, res) + if read_path.endswith("conf/config.sh"): + update_hive_metastore(read_path) + file_data = file_read(read_path) + res = file_data.replace("172.17.0.5", ip) + file_write(read_path, res) + + +def read_file_yml(ip): + for yaml_path in yml_file_list: + file = open(yaml_path, 'rb') + data = yaml.load(file, Loader=yaml.Loader) + + if data.get("eureka").get("client").get("serviceUrl").get("defaultZone"): + data["eureka"]["client"]["serviceUrl"]["defaultZone"] = "http://{}:9600/eureka/".format(str(ip)) + file.close() + + with open(yaml_path, 'w', encoding='utf8') as f: + yaml.dump(data=data, stream=f, allow_unicode=True) + + +def update_hive_metastore(file_path): + config = configparser.ConfigParser() + config.read_file(open('/wedatasphere/docker/conf/conf.ini', encoding="utf-8-sig")) + + host = config.get("hiveMetaData", "hive.meta.host") + db = config.get("hiveMetaData", "hive.meta.db") + username = config.get("hiveMetaData", "hive.meta.user") + pwd = config.get("hiveMetaData", "hive.meta.password") + + file_data = file_read(file_path) + res = file_data.replace("UserHive", username).replace("123456789", pwd).replace("HiveMetastore", db). \ + replace("172.16.13.224", host) + file_write(file_path, res) + + +def read_file_pro(ip): + for pro_file in pro_file_list: + if not pro_file.endswith("linkis-ps-publicservice.properties"): + if pro_file.endswith("host.properties"): + hostname = socket.gethostname() + file_data = file_read(pro_file) + res = file_data.replace("dss.node.cn", hostname) + file_write(pro_file, res) + file_data = file_read(pro_file) + res = file_data.replace("172.17.0.5", ip).replace("172.16.13.131", ip) + file_write(pro_file, res) + else: + update_hive_metastore(pro_file) + + +def run(): + # 获取IP地址 + ip = get_host_ip() + # 文件分类 + file_types() + read_file_sh(ip) + read_file_yml(ip) + read_file_pro(ip) + print("配置文件更新成功") + + +if __name__ == "__main__": + run() diff --git a/dockers/cdh6.3.2/run_docker_dss_server.sh b/dockers/cdh6.3.2/run_docker_dss_server.sh new file mode 100644 index 0000000..973e8c5 --- /dev/null +++ b/dockers/cdh6.3.2/run_docker_dss_server.sh @@ -0,0 +1,19 @@ +# 启动容器 +docker run --name dss_linkis_server_v1 \ +-h dss.node.cn \ +-e HOST_IP=172.16.13.133 \ +-v /opt/cloudera/parcels/CDH:/wedatasphere/cdh \ +-v /opt/cloudera/parcels:/opt/cloudera/parcels \ +-v /usr/local/service/spark3:/opt/service/spark3 \ +-v /opt/cloudera/parcels/FLINK:/opt/service/flink \ +-v /etc/hadoop/conf:/etc/hadoop/conf \ +-v /etc/sqoop/conf:/etc/sqoop/conf \ +-v /etc/hive/conf:/etc/hive/conf \ +-p 3306:3306 \ +-p 8085:8085 \ +-p 9600:9600 \ +-p 8060:8060 \ +-p 8090:8090 \ +-p 9098:9098 \ +-p 8055:8055 \ +-itd dss_linkis_server_test \ No newline at end of file diff --git a/dockers/cdh6.3.2/sbin/start-all.sh b/dockers/cdh6.3.2/sbin/start-all.sh new file mode 100644 index 0000000..a569d78 --- /dev/null +++ b/dockers/cdh6.3.2/sbin/start-all.sh @@ -0,0 +1,173 @@ +#!/bin/sh +#Actively load user env + +#source ~/.bash_profile + +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +export LINKIS_DSS_HOME=`cd ${shellDir}/../install/dss_linkis;pwd` + +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 +else + echo "Succeed to" + $1 +fi +} + +function isSuccessTest(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + tail -f /var/log/lastlog + #exit 1 +else + echo "Succeed to" + $1 + tail -f /var/log/lastlog +fi +} + +source ${LINKIS_DSS_HOME}/conf/config.sh +source ${LINKIS_DSS_HOME}/conf/db.sh +source ${LINKIS_DSS_HOME}/bin/replace.sh +#######设置为当前路径，如果不需要直接注掉这执行函数########## +setCurrentRoot + +echo "########################################################################" +echo "###################### Begin to start Linkis ###########################" +echo "########################################################################" +sh ${LINKIS_HOME}/sbin/linkis-start-all.sh +isSuccess "start Linkis" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start DSS Service ######################" +echo "########################################################################" +sh ${DSS_HOME}/sbin/dss-start-all.sh +isSuccess "start DSS Service" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start DSS & Linkis web ##########################" +echo "########################################################################" + +dss_ipaddr=$(hostname -I) +dss_ipaddr=${dss_ipaddr// /} + +function startDssWeb(){ +version=`cat /etc/redhat-release|sed -r 's/.* ([0-9]+)\..*/\1/'` +# centos 6 +if [[ $version -eq 6 ]]; then + # sudo /etc/init.d/nginx start + isSuccess "start DSS & Linkis Web" +fi + +# centos 7 +if [[ $version -eq 7 ]]; then + # sudo systemctl start nginx + isSuccess "start DSS & Linkis Web" +fi +} +startDssWeb + +if [[ $LINKIS_EUREKA_INSTALL_IP == "127.0.0.1" ]] || [ -z "$LINKIS_EUREKA_INSTALL_IP" ]; then + LINKIS_EUREKA_INSTALL_IP=${dss_ipaddr} +fi + +echo "" +echo "===============================================" +echo "There are eight micro services in Linkis：" +echo "linkis-cg-engineconnmanager" +echo "linkis-cg-engineplugin" +echo "linkis-cg-entrance" +echo "linkis-cg-linkismanager" +echo "linkis-mg-eureka" +echo "linkis-mg-gateway" +echo "linkis-ps-cs" +echo "linkis-ps-publicservice" +echo "-----------------------------------------------" +echo "There are six micro services in DSS：" +echo "dss-framework-project-server" +echo "dss-framework-orchestrator-server-dev" +echo "dss-workflow-server-dev" +echo "dss-flow-entrance" +echo "dss-datapipe-server" +echo "dss-apiservice-server" +echo "===============================================" +echo "" +echo "Log path of Linkis: linkis/logs" +echo "Log path of DSS : dss/logs" + +echo "" +echo "You can check DSS & Linkis by acessing eureka URL: http://${LINKIS_EUREKA_INSTALL_IP}:${LINKIS_EUREKA_PORT}" +echo "You can acess DSS & Linkis Web by http://${dss_ipaddr}:${DSS_WEB_PORT}" +echo "" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Qualitis##########################" +echo "########################################################################" + +export QUALITIS_HOME=`cd ${shellDir}/../install/qualitis;pwd` + +sh ${QUALITIS_HOME}/bin/start.sh +isSuccess "start Qualitis" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Exchangis#########################" +echo "########################################################################" + +export EXCHANGIS_HOME=`cd ${shellDir}/../install/exchangis;pwd` + +sh ${EXCHANGIS_HOME}/sbin/daemon.sh start server +isSuccess "start Exchangis" + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Visualis##########################" +echo "########################################################################" + +export VISUALIS_HOME=`cd ${shellDir}/../install/visualis;pwd` + +sh ${VISUALIS_HOME}/bin/start-visualis-server.sh +isSuccess "start Visualis" + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Schedulis##########################" +echo "########################################################################" + +export SCHEDULIS_HOME=`cd ${shellDir}/../install/schedulis;pwd` + +cd ${SCHEDULIS_HOME}/schedulis_0.7.1_exec && sh bin/start-exec.sh +isSuccess "start SchedulisExec" + +cd ${SCHEDULIS_HOME}/schedulis_0.7.1_web && sh bin/start-web.sh +isSuccess "start SchedulisWeb" + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Streamis##########################" +echo "########################################################################" + +export STREAMIS_HOME=`cd ${shellDir}/../install/streamis/streamis-server;pwd` + +sh ${STREAMIS_HOME}/bin/start-streamis-server.sh +isSuccessTest "start Streamis" + diff --git a/dockers/cdh6.3.2/sbin/stop-all.sh b/dockers/cdh6.3.2/sbin/stop-all.sh new file mode 100644 index 0000000..84274ca --- /dev/null +++ b/dockers/cdh6.3.2/sbin/stop-all.sh @@ -0,0 +1,123 @@ +#!/bin/sh +#Actively load user env + +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +export LINKIS_DSS_HOME=`cd ${shellDir}/../install/dss_linkis;pwd` + +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + # exit 1 +else + echo "Succeed to" + $1 +fi +} + +source ${LINKIS_DSS_HOME}/conf/config.sh +source ${LINKIS_DSS_HOME}/conf/db.sh +source ${LINKIS_DSS_HOME}/bin/replace.sh +#######设置为当前路径，如果不需要直接注掉这执行函数########## +setCurrentRoot + +echo "########################################################################" +echo "###################### Begin to stop Linkis ############################" +echo "########################################################################" +sh ${LINKIS_HOME}/sbin/linkis-stop-all.sh +isSuccess "stop Linkis" + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to stop DSS Service #######################" +echo "########################################################################" +sh ${DSS_HOME}/sbin/dss-stop-all.sh +isSuccess "stop DSS Service" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to stop DSS & Linkis Web ##################" +echo "########################################################################" +function stopDssWeb(){ +version=`cat /etc/redhat-release|sed -r 's/.* ([0-9]+)\..*/\1/'` +# centos 6 +if [[ $version -eq 6 ]]; then + # sudo /etc/init.d/nginx stop + isSuccess "stop DSS & Linkis Web" +fi + +# centos 7 +if [[ $version -eq 7 ]]; then + # sudo systemctl stop nginx + isSuccess "stop DSS & Linkis Web" +fi +} +stopDssWeb + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Qualitis##########################" +echo "########################################################################" + +export QUALITIS_HOME=`cd ${shellDir}/../install/qualitis;pwd` + +sh ${QUALITIS_HOME}/bin/stop.sh +isSuccess "stop Qualitis" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Exchangis#########################" +echo "########################################################################" + +export EXCHANGIS_HOME=`cd ${shellDir}/../install/exchangis;pwd` + +sh ${EXCHANGIS_HOME}/sbin/daemon.sh stop server +isSuccess "stop Exchangis" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Visualis##########################" +echo "########################################################################" + +export VISUALIS_HOME=`cd ${shellDir}/../install/visualis;pwd` + +sh ${VISUALIS_HOME}/bin/stop-visualis-server.sh +isSuccess "stop Visualis" + + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Schedulis#########################" +echo "########################################################################" + +export SCHEDULIS_HOME=`cd ${shellDir}/../install/schedulis;pwd` + + +cd ${SCHEDULIS_HOME}/schedulis_0.7.1_exec && sh bin/shutdown-exec.sh +isSuccess "start SchedulisExec" + +cd ${SCHEDULIS_HOME}/schedulis_0.7.1_web && sh bin/shutdown-web.sh +isSuccess "start SchedulisWeb" + +echo "" +echo "" +echo "########################################################################" +echo "###################### Begin to start Streamis##########################" +echo "########################################################################" + +export STREAMIS_HOME=`cd ${shellDir}/../install/streamis/streamis-server;pwd` + +sh ${STREAMIS_HOME}/bin/stop-streamis-server.sh +isSuccess "stop Streamis" \ No newline at end of file diff --git a/dockers/cdh6.3.2/sbin/wedatasphere-env.sh b/dockers/cdh6.3.2/sbin/wedatasphere-env.sh new file mode 100644 index 0000000..2ec0615 --- /dev/null +++ b/dockers/cdh6.3.2/sbin/wedatasphere-env.sh @@ -0,0 +1,8 @@ +#!/bin/sh +#Actively load user env + +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi diff --git "a/docs/dockers/cdh6.3.2/\345\215\207\347\272\247\346\214\207\345\215\227.md" "b/docs/dockers/cdh6.3.2/\345\215\207\347\272\247\346\214\207\345\215\227.md" new file mode 100644 index 0000000..c704fa2 --- /dev/null +++ "b/docs/dockers/cdh6.3.2/\345\215\207\347\272\247\346\214\207\345\215\227.md" @@ -0,0 +1,58 @@ +用户如何升级或替换WDS的某个组件 +1、修改Dockerfile 文件 +``` +FROM 基础镜像 + +RUN pip3 install --no-cache-dir -r /wedatasphere/docker/conf/requirements.txt -i \ + http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + +COPY ./docker /wedatasphere/docker/ +COPY ./sbin /wedatasphere/sbin/ + +RUN chown -R hdfs:hdfs /wedatasphere + +RUN chmod +x /wedatasphere/docker/script/run.sh +WORKDIR /wedatasphere + +# 把这行执行一键启动的命令注释掉 +# CMD ["/bin/bash", "/wedatasphere/docker/script/run.sh"] +``` +重新构建镜像并启动 +``` +docker build -t 镜像名 . + +docker run -itd 镜像名 /sbin/init +``` +2、升级或替换组件 + 2.1 备份需要升级的旧的组件包 + ``` + # 进入容器 + docker exec -it 镜像名 /bin/bash + # 备份组件包 + cd /wedatasphere/install/ + mv linkis linkis_bak + +``` + 2.2 将需要替换的包cp到镜像中 + ``` + docker cp 组件包 镜像名:/wedatasphere/install/ + ``` + +3、重新打包基础镜像 +``` +docker commit 容器id 镜像名 +``` +修改Dockerfile文件中的FROM 为刚重新打包的镜像 + +4、构建镜像 +``` +docker bulid -t dss_linkis_server . +``` +5、运行容器 + ``` + sh run_docker_dss_server.sh +``` +6、验证是否升级成功 +浏览器访问地址 http://宿主机IP:8085/#/login 即可登录系统UI,登录验证升级的组件是否正常即可。 + +注：由于版本兼容问题，升级可能会导致部分功能或者是服务不可用，升级前请先确认官方是否对该版本已经兼容， 参考[DataSphereStudio](https://github.com/WeBankFinTech/DataSphereStudio/blob/master/README-ZH.md#%E5%9B%9B%E5%B7%B2%E9%9B%86%E6%88%90%E7%9A%84%E6%95%B0%E6%8D%AE%E5%BA%94%E7%94%A8%E7%BB%84%E4%BB%B6)。 \ No newline at end of file diff --git "a/docs/dockers/cdh6.3.2/\345\256\211\350\243\205\351\203\250\347\275\262\346\226\207\346\241\243.md" "b/docs/dockers/cdh6.3.2/\345\256\211\350\243\205\351\203\250\347\275\262\346\226\207\346\241\243.md" new file mode 100644 index 0000000..31b6405 --- /dev/null +++ "b/docs/dockers/cdh6.3.2/\345\256\211\350\243\205\351\203\250\347\275\262\346\226\207\346\241\243.md" @@ -0,0 +1,135 @@ +**先决条件**： +1. 安装机上需要有cdh6.3.2的环境，cdh环境中还需要有flink环境。 +2. docker环境。 + +参考文章： + +[CM+CDH6.3.2环境搭建](https://www.cnblogs.com/ttzzyy/p/13072774.html) + +[flink编译parcel包并集成至cdh6.3.2](https://blog.csdn.net/cn987654/article/details/117516124?spm=1001.2101.3001.6650.17&utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7EBlogCommendFromBaidu%7ERate-17-117516124-blog-123007848.pc_relevant_aa&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7EBlogCommendFromBaidu%7ERate-17-117516124-blog-123007848.pc_relevant_aa&utm_relevant_index=22) + +[Docker安装教程](https://www.jianshu.com/p/946e703df75d?v=1678933661213) + +以上环境没有的同学可以参考上面的文章安装或者自行百度安装。 + + +1、 版本说明： +1. WeDataSphere组件版本： + +| 组件名 | Apache Linkis | DataSphere Studio | Schedulis | Qualitis | Exchangis | Visualis | Streamis | MYSQL | JDK | +|-------|---------------|-------------------|-----------|----------|-----------|----------|----------|--------|-----| +| 版本号 | 1.3.1 | 1.1.1 | 0.7.1 | 0.9.2 | 1.0.0 | 1.0.0 | 0.2.0 | 5.1.49 | 1.8 | + + +2. CDH6.3.2 版本 + +| 组件名 | Hadoop | Hive | Spark | Flink | Sqoop | Trino | +|-------|----------------|----------------|-------|--------|-------|-------| +| 版本号 | 3.0.0-cdh6.3.2 | 2.1.1-cdh6.3.2 | 3.0.0 | 1.12.4 | 1.4.6 | 371 | + +2、github 拉取代码 + +``` +git clone https://github.com/WeBankFinTech/WeDataSphere.git +git chechout wds-cdh-6.3.2 + +``` +3、下载镜像 + + 由于镜像包过大，上传到dockerhub太慢，所以把安装包上传到oss上供大家下载使用 + + 3.1 下载镜像包: + ``` + wget https://kevin-1251411289.cos.ap-beijing.myqcloud.com/public/dss_linkis.tar + ``` + 3.2 使用load方法加载刚才下载的tar文件 + ``` + docker load < ./dss_linkis.tar + ``` + 3.3 在新的机器上再此使用docker images命令查看本机的镜像，检查刚才load的镜像有没有加载进来 + ``` + # 查看镜像 + docker images + # 若发现加载进来的镜像名称、标签均为none，使用docker tag iamges_id 镜像名:标签名， 修改即可。 + ``` + 3.4 加载成功后参考安装部署文档运行镜像即可。 + +4、修改Dockerfile文件 + +**若上步未修改拉取的镜像名则此步可跳过** +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303241122062.png) +将上图红框中的内容修改成 上一步 下载到本地的镜像即可。 + + + +5、修改conf 文件，配置自己的cdh中hive元数据的地址 + +cd dockers/cdh6.3.2/docker/conf/ 目录下： +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161054156.png) + +找到conf.ini文件，修改成自己cdh中hive元数据的数据库信息 +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161045541.png) +6、构建镜像 + +cd dockers/cdh6.3.2 执行构建 +``` +docker build -t dss_linkis_server . +``` +执行后等待构建成功即可。 + +7、修改docker 启动文件 + +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161056381.png) +vim run_docker_dss_server.sh +``` +# 启动容器 +docker run \ +# 容器名称 +--name dss_linkis_server_v1 \ +# 容器hostname 固定不用改动 +-h dss.node.cn \ +# 设置环境变量，需要设置成自己的宿主机ip +-e HOST_IP=172.16.13.133 \ +# 目录挂载，需要修改成自己机器上的路径 +# 本地cdh环境挂载到容器中 +-v /opt/cloudera/parcels/CDH:/wedatasphere/cdh \ +-v /opt/cloudera/parcels:/opt/cloudera/parcels \ +# 本地spark3挂载到容器中 +-v /usr/local/service/spark3:/opt/service/spark3 \ +# 本地flink挂载到容器中 +-v /opt/cloudera/parcels/FLINK:/opt/service/flink \ +# 本地hadoop相关conf目录挂载 +-v /etc/hadoop/conf:/etc/hadoop/conf \ +-v /etc/sqoop/conf:/etc/sqoop/conf \ +-v /etc/hive/conf:/etc/hive/conf \ +# 端口映射 +-p 3306:3306 \ +-p 8085:8085 \ +-p 9600:9600 \ +-p 8060:8060 \ +-p 8090:8090 \ +-p 9098:9098 \ +-p 8055:8055 \ +-itd \ +# 上步构建时用的image名称 +dss_linkis_server +``` +8、运行docker +``` +sh run_docker_dss_server.sh +``` + +9、查看是否运行成功 + +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161133229.png) + +查看docker日志看各服务是否启动 +``` +docker logs CONTAINER_ID -f +``` +10、登录 + +浏览器访问地址 http://宿主机IP:8085/#/login 即可登录系统UI。 +默认的用户名和密码是:hdfs/hdfs +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161139967.png) + diff --git "a/docs/dockers/cdh6.3.2/\345\270\270\350\247\201\351\227\256\351\242\230.md" "b/docs/dockers/cdh6.3.2/\345\270\270\350\247\201\351\227\256\351\242\230.md" new file mode 100644 index 0000000..51b066d --- /dev/null +++ "b/docs/dockers/cdh6.3.2/\345\270\270\350\247\201\351\227\256\351\242\230.md" @@ -0,0 +1,24 @@ +问题二： +``` +ERROR jobRequest(IDE_hdfs_flink_0)  execute failed,21304, Task is Failed,errorMsg: errCode: 12003 ,desc: dss.node.cn:9101_0 Failed  to async get EngineNode AMErrorException: errCode: 30002 ,desc: ServiceInstance(linkis-cg-engineconn, dss.node.cn:41684) ticketID:e0f49a8c-e9b5-4cc1-b7f8-9684e025ccde 初始化引擎失败,原因: ServiceInstance(linkis-cg-engineconn, dss.node.cn:41684): log dir: /wedatasphere/appcom/tmp/hdfs/20230315/flink/e0f49a8c-e9b5-4cc1-b7f8-9684e025ccde/logs,FileNotFoundException: File /wedatasphere/service/flink/lib/flink-dist_2.11-1.12.2.jar does not exist ,ip: dss.node.cn ,port: 9101 ,serviceKind: linkis-cg-linkismanager ,ip: dss.node.cn ,port: 9104 ,serviceKind: linkis-cg-entrance +``` +找到flink的安装目录 find . -name "flink-dist_2.11*" 找的jar报所在位置，cp *.jar 到 /wedatasphere/service/flink/lib/目录下 +如果还报找不到这个包的话，查看你的flink版本是否和报错的版本一致，不一致的修改成报错提示的这个版本。 + + +问题二： +如果shdulis 服务打开发现是乱码 +需要进入容器重启服务即可。 + + +其他问题参考： + +[WeDataSphere 常见问题(含DSS，Linkis等)QA文档](https://docs.qq.com/doc/DSGZhdnpMV3lTUUxq) + +[单机部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/deploy-quick) + +[Flink引擎相关问题汇总](https://mp.weixin.qq.com/s/VxZ16IPMd1CvcrvHFuU4RQ) + +[JDBC引擎相关问题汇总](https://mp.weixin.qq.com/s?__biz=MzUzMjMwNzE0MQ==&mid=2247488632&idx=1&sn=e096e362630d1d8e6430e03eb4b2e736&chksm=fab419d2cdc390c48e675743d76c86d79d28bf0c8e1521f19134e585b8ebada54816bbffe15d&scene=178&cur_album_id=1959159714886270978#rd) + +[CDH5环境中的试用记录](https://mp.weixin.qq.com/s?__biz=MzUzMjMwNzE0MQ==&mid=2247488397&idx=1&sn=6af0c5f503ad7d17dba5f7c0997f8c3d&chksm=fab41e27cdc397319a7f0e45b39fbd8dd65f44adef741c4aff244b0e68967e56434e9f70cd9d&cur_album_id=1959159714886270978&scene=189#wechat_redirect) \ No newline at end of file diff --git "a/docs/dockers/cdh6.3.2/\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/dockers/cdh6.3.2/\345\274\200\345\217\221\346\226\207\346\241\243.md" new file mode 100644 index 0000000..65548d3 --- /dev/null +++ "b/docs/dockers/cdh6.3.2/\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -0,0 +1,57 @@ +1、目录结构 + +├── wedatasphere --根目录 + +│ ├── sbin --wedatasphere全家桶一键启动和一键停止的脚本目录 + +│ │ ├── start-all.sh --wedatasphere全家桶一键启动脚本 + +│ │ ├── stop-all.sh --wedatasphere全家桶一键停止脚本 + +│ │ ├── wedatasphere-env.sh --wedatasphere全家桶环境变量配置脚本 + +│ ├── install --wedatasphere各组件安装包的存放目录 + +│ │ ├── --dss_linkis Linkis&dss安装包根目录 + +│ │ ├── …… 其他组件包安装目录 + +│ ├── config --wedatasphere各组件配置文件的存放目录 + +│ │ ├── linkis-config --Linkis 配置文件根目录 + +│ │ ├── …… + +│ ├── logs --wedatasphere各组件日志文件的存放目录 + +│ │ ├── linkis -- Linkis日志文件根目录 + +│ │ ├── …… + +2、启动逻辑 + +找到目录中的wedatasphere全家桶一键启动脚本 查看主要代码逻辑： +``` +# 设置组件的HOME目录并cd到此目录中 +shellDir=`dirname $0` +export LINKIS_DSS_HOME=`cd ${shellDir}/../install/dss_linkis;pwd` + +# 执行相关组件的启动脚本即可 +sh ${STREAMIS_HOME}/bin/start-streamis-server.sh +isSuccessTest "start Streamis" + +``` + +3、如何新增一个组件 + +3.1 修改Dockerfile文件，将需要的组件copy到容器中 + +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161333426.png) + +3.2 修改一键启动脚本将组件的启动命令放到最后，对应的停止脚本也需要写到一键停止脚本中 +![image.png](https://cdn.jsdelivr.net/gh/MrZsy/noteImage@main/img/202303161337153.png) + +3.3 进入容器启动dss&linkis服务，具体每个组件操作参考官方文档中的操作说明即可。 + + +**其他组件具体开发文档参考官方[开发文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%BC%80%E5%8F%91%E6%96%87%E6%A1%A3)** \ No newline at end of file diff --git "a/docs/dockers/cdh6.3.2/\351\225\234\345\203\217\344\270\213\350\275\275.md" "b/docs/dockers/cdh6.3.2/\351\225\234\345\203\217\344\270\213\350\275\275.md" new file mode 100644 index 0000000..a10dd6e --- /dev/null +++ "b/docs/dockers/cdh6.3.2/\351\225\234\345\203\217\344\270\213\350\275\275.md" @@ -0,0 +1,20 @@ + +下载镜像方法： + +由于镜像包过大，上传到dockerhub太慢，所以把安装包上传到oss上供大家下载使用 + +1、下载镜像包: +``` +wget https://kevin-1251411289.cos.ap-beijing.myqcloud.com/public/dss_linkis.tar +``` +2、使用load方法加载刚才下载的tar文件 +``` +docker load < ./dss_linkis.tar +``` +3、在新的机器上再此使用docker images命令查看本机的镜像，检查刚才load的镜像有没有加载进来 +``` +# 查看镜像 +docker images +# 若发现加载进来的镜像名称、标签均为none，使用docker tag iamges_id 镜像名:标签名， 修改即可。 +``` +4、加载成功后参考安装部署文档运行镜像即可。 \ No newline at end of file