-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] Provide init tools (#3999)
Co-authored-by: Zzm0809 <[email protected]> Co-authored-by: GH Action - Upstream Sync <[email protected]>
- Loading branch information
3 people
authored
Dec 3, 2024
1 parent
44c1066
commit 7bc8913
Showing
9 changed files
with
465 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/bin/bash | ||
|
|
||
| EXTERNAL_CONNECTIVITY_CHECK_URL="www.baidu.com" | ||
|
|
||
| echo -e "${YELLOW}Checking whether your network can connect to the Internet properly (${EXTERNAL_CONNECTIVITY_CHECK_URL}) ...${RESET}" | ||
| if ! ping -c 1 ${EXTERNAL_CONNECTIVITY_CHECK_URL} &> /dev/null; then | ||
| echo -e "${RED}Your network cannot connect to the Internet using ping mode, please check whether your network environment is normal, and the program will try to use curl mode to detect the network connection again...${RESET}" | ||
| if ! curl -I -s --connect-timeout 5 ${EXTERNAL_CONNECTIVITY_CHECK_URL} -w '%{http_code}' | tail -n1 | grep "200" &> /dev/null; then | ||
| echo -e "${RED}Your network cannot be connected to the Internet using curl mode, please check whether your network environment is normal。${RESET}" | ||
| echo -e "${YELLOW}Note that in some network environments, firewalls or security policies may block ICMP requests (i.e., pings). If this happens, you can use curl to detect the network connection.${RESET}" | ||
| exit 1 | ||
| else | ||
| echo -e "${GREEN}Your network can use curl to connect to the Internet and proceed to the next step...${RESET}" | ||
| fi | ||
| else | ||
| echo -e "${GREEN}Your network can use ping to connect to the Internet and proceed to the next step...${RESET}" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| #!/bin/bash | ||
|
|
||
| DINKY_HOME=$1 | ||
|
|
||
| echo -e "${GREEN}====================== The database configuration file is initialized ======================${RESET}" | ||
|
|
||
| if [ -z "$DINKY_HOME" ]; then | ||
| echo -e "${RED}The parameter is wrong, please check!${RESET}" | ||
| exit 1 | ||
| fi | ||
|
|
||
| while true; do | ||
| read -p "Please select a database type (1.MySQL 2.PostgresSQL)：" db_type | ||
| read -p "Please enter the database address (hostname or IP, default localhost): " -i "localhost" db_host | ||
| read -p "Please enter the database port (default 3306): " -i 3306 db_port | ||
| read -p "Please enter a database name (default dinky): " -i "dinky" db_name | ||
| read -p "Please enter the database username (default dinky):" -i "dinky" db_username | ||
| read -s -p "Please enter the database password (default dinky):" -i "dinky" db_password | ||
| echo | ||
|
|
||
| db_type=$(echo "$db_type" | tr -d '[:space:]') | ||
| db_host=$(echo "$db_host" | tr -d '[:space:]') | ||
| db_port=$(echo "$db_port" | tr -d '[:space:]') | ||
| db_name=$(echo "$db_name" | tr -d '[:space:]') | ||
| db_username=$(echo "$db_username" | tr -d '[:space:]') | ||
| db_password=$(echo "$db_password" | tr -d '[:space:]') | ||
|
|
||
| case $db_type in | ||
| 1) | ||
| echo -e "${YELLOW}Configuring MySQL database related information...${RESET}" | ||
| config_file="${DINKY_HOME}/config/application-mysql.yml" | ||
| echo -e "${GREEN} The automatic initialization script uses the export environment variable method to support the loading of environment variables of the data source. The configuration file is：${config_file} ${RESET}" | ||
| echo "export DB_ACTIVE=mysql" >> /etc/profile | ||
| echo "export MYSQL_ADDR=${db_host}:${db_port}" >> /etc/profile | ||
| echo "export MYSQL_DATABASE=${db_name}" >> /etc/profile | ||
| echo "export MYSQL_USERNAME=${db_username}" >> /etc/profile | ||
| echo "export MYSQL_PASSWORD=${db_password}" >> /etc/profile | ||
| source /etc/profile | ||
|
|
||
| echo -e "${GREEN}MySQLThe configuration of database related information is completed. Please confirm whether the following configuration is correct：${RESET}" | ||
| grep -E '^(export DB_ACTIVE|export MYSQL_ADDR|export MYSQL_DATABASE|export MYSQL_USERNAME|export MYSQL_PASSWORD)' /etc/profile | grep -v "^#" | grep -v "^$" | ||
| break | ||
| ;; | ||
| 2) | ||
| echo -e "${YELLOW}Configuring PostgresSQL database related information...${RESET}" | ||
| config_file="${DINKY_HOME}/config/application-pgsql.yml" | ||
|
|
||
| echo -e "${GREEN}The automatic initialization script uses the export environment variable method to support the loading of environment variables from the data source configuration file. The configuration file is：${config_file} ${RESET}" | ||
|
|
||
| echo "export DB_ACTIVE=pgsql" >> /etc/profile | ||
| echo "export POSTGRES_ADDR=${db_host}:${db_port}" >> /etc/profile | ||
| echo "export POSTGRES_DB=${db_name}" >> /etc/profile | ||
| echo "export POSTGRES_USER=${db_username}" >> /etc/profile | ||
| echo "export POSTGRES_PASSWORD=${db_password}" >> /etc/profile | ||
| source /etc/profile | ||
|
|
||
| echo -e "${GREEN}PostgresSQL The configuration of database related information is completed. Please confirm whether the following configuration is correct：${RESET}" | ||
| grep -E '^(export DB_ACTIVE|export POSTGRES_ADDR|export POSTGRES_DB|export POSTGRES_USER|export POSTGRES_PASSWORD)' /etc/profile | grep -v "^#" | grep -v "^$" | ||
| break | ||
| ;; | ||
| *) | ||
| echo -e "${RED}The entered database type is incorrect, please select the correct database type again.${RESET}" | ||
| ;; | ||
| esac | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| #!/bin/bash | ||
|
|
||
| CURRENT_FLINK_FULL_VERSION=$1 | ||
| FLINK_VERSION_SCAN=$2 | ||
| DINKY_TMP_DIR=$3 | ||
| EXTENDS_HOME=$4 | ||
| DINKY_HOME=$5 | ||
|
|
||
| echo -e "${GREEN}====================== Flink dependency initialization ======================${RESET}" | ||
|
|
||
| echo -e "${BLUE}Parameters: The current Flink version is：${CURRENT_FLINK_FULL_VERSION}，The scanned Flink version is：${FLINK_VERSION_SCAN} ，The temporary directory is：${DINKY_TMP_DIR} ，The expansion package directory is：${EXTENDS_HOME} ，Dinky The root directory is：${DINKY_HOME}${RESET}" | ||
|
|
||
| if [ -z "$CURRENT_FLINK_FULL_VERSION" ] || [ -z "$FLINK_VERSION_SCAN" ] || [ -z "$DINKY_TMP_DIR" ] || [ -z "$EXTENDS_HOME" ] || [ -z "$DINKY_HOME" ]; then | ||
| echo -e "${RED}Parameter error, please check!${RESET}" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -f "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then | ||
| echo -e "${YELLOW}$DINKY_TMP_DIR ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file，To ensure completeness, delete first ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}" | ||
| rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz | ||
| if [ -d "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then | ||
| echo -e "${YELLOW}The flink directory already exists, delete it $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" | ||
| rm -rf $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION} | ||
| fi | ||
| fi | ||
|
|
||
| try_tsinghua_mirror() { | ||
| local tsinghua_url="https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" | ||
| local apache_url="https://archive.apache.org/dist/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" | ||
|
|
||
| echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${DINKY_TMP_DIR} directory${RESET}" | ||
| if download_file "$tsinghua_url" "$DINKY_TMP_DIR"; then | ||
| echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${tsinghua_url}${RESET}" | ||
| return 0 | ||
| else | ||
| echo -e "${YELLOW}File not found in Tsinghua University mirror, try downloading from Apache official source...${RESET}" | ||
| if download_file "$apache_url" "$DINKY_TMP_DIR"; then | ||
| echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${apache_url}${RESET}" | ||
| return 0 | ||
| else | ||
| echo -e "${RED}Downloading from Apache official source also failed, please check the network or download manually。${RESET}" | ||
| return 1 | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| if ! try_tsinghua_mirror; then | ||
| exit 0 | ||
| fi | ||
|
|
||
|
|
||
| echo -e "${GREEN}Flink installation package download completed。${RESET}" | ||
| echo -e "\n${GREEN}===============================================================${RESET}\n" | ||
| echo -e "${GREEN}Start decompressing the Flink installation package...${RESET}" | ||
| tar -zxvf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${DINKY_TMP_DIR}/ | ||
| if [ $? -eq 0 ]; then | ||
| echo -e "${GREEN}Flink installation package decompression completed。${RESET}" | ||
| else | ||
| echo -e "${RED}Flink installation package failed to decompress, please check。${RESET}" | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo -e "\n${GREEN}===============================================================${RESET}\n" | ||
|
|
||
| flink_dir_tmp=$(ls -n ${DINKY_TMP_DIR} | grep '^d' | grep flink | awk '{print $9}') | ||
| full_flink_dir_tmp="${DINKY_TMP_DIR}/${flink_dir_tmp}" | ||
| echo -e "${BLUE}Unzipped directory name：${full_flink_dir_tmp}${RESET}" | ||
|
|
||
|
|
||
|
|
||
| echo -e "${GREEN}Process ${full_flink_dir_tmp}/lib/flink-table-planner-loader* file...${RESET}" | ||
| rm -rf ${full_flink_dir_tmp}/lib/flink-table-planner-loader* | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process ${full_flink_dir_tmp}/opt/flink-table-planner_2.12-*.jar file...${RESET}" | ||
| mv ${full_flink_dir_tmp}/opt/flink-table-planner_2.12-*.jar ${full_flink_dir_tmp}/lib/ | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process flink jar dependencies into dinky...${RESET}" | ||
| cp -r ${full_flink_dir_tmp}/lib/*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
| echo -e "${GREEN}jar dependency processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process flink-sql-client ...${RESET}" | ||
| cp -r ${full_flink_dir_tmp}/opt/flink-sql-client-*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process flink-cep-scala ...${RESET}" | ||
| cp -r ${full_flink_dir_tmp}/opt/flink-cep-scala*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process flink-queryable-state-runtime ...${RESET}" | ||
| cp -r ${full_flink_dir_tmp}/opt/flink-queryable-state-runtime*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN}Process flink-state-processor-api ...${RESET}" | ||
| cp -r ${full_flink_dir_tmp}/opt/flink-state-processor-api*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
| echo -e "${GREEN}Processing completed。${RESET}" | ||
|
|
||
| echo -e "${GREEN} ================= List files in the ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ directory ==============${RESET}" | ||
| ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ | ||
|
|
||
| echo -e "${YELLOW}Please check the above dependent files。${RESET}" | ||
|
|
||
| echo -e "${GREEN}The basic dependency processing is completed, please perform subsequent operations according to the actual situation.${RESET}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| #!/bin/bash | ||
|
|
||
| EXTENDS_HOME=$1 | ||
|
|
||
|
|
||
| echo -e "${GREEN}====================== Hadoop dependency initialization ======================${RESET}" | ||
|
|
||
| read -p "Please select the Hadoop-uber version to download (enter 2 or 3):" hadoop_uber_version | ||
| hadoop_uber_version=$(echo "$hadoop_uber_version" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') | ||
|
|
||
| case $hadoop_uber_version in | ||
| 2) | ||
| echo -e "${YELLOW}Start downloading Hadoop-uber 2 version package...${RESET}" | ||
| if [ -f "$EXTENDS_HOME/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" ]; then | ||
| echo -e "${YELLOW}The flink-shaded-hadoop-2-uber-2.8.3-10.0.jar file already exists and there is no need to download it again.${RESET}" | ||
| else | ||
| download_url="https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" | ||
| download_file "$download_url" "$EXTENDS_HOME" | ||
| fi | ||
| ;; | ||
| 3) | ||
| if [ -f "$EXTENDS_HOME/flink-shaded-hadoop-3-uber-3.1.1.7.2.9.0-173-9.0.jar" ]; then | ||
| echo -e "${YELLOW}The flink-shaded-hadoop-3-uber-3.1.1.7.2.9.0-173-9.0.jar file already exists and there is no need to download it again.${RESET}" | ||
| else | ||
| echo -e "${YELLOW}Start downloading Hadoop-uber 3 version package...${RESET}" | ||
| download_url="https://repository.cloudera.com/artifactory/cloudera-repos/org/apache/flink/flink-shaded-hadoop-3-uber/3.1.1.7.2.9.0-173-9.0/flink-shaded-hadoop-3-uber-3.1.1.7.2.9.0-173-9.0.jar" | ||
| download_file "$download_url" "$EXTENDS_HOME" | ||
| fi | ||
| ;; | ||
| *) | ||
| echo -e "${RED}The entered version number is incorrect, please re-run the script to select the correct version.${RESET}" | ||
| ;; | ||
| esac | ||
|
|
||
| echo -e "${GREEN}After the download is completed, subsequent installation and configuration operations can be performed as needed.${RESET}" |
Oops, something went wrong.