From 2a45e6e36494ee209e42e217678606ff70cdf21b Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Fri, 4 Jan 2019 13:37:04 -0700 Subject: [PATCH 01/19] Removing version from build script, and adding MongoDB restart command (might need to rearrange so it pops more) --- .gitignore | 2 +- dev-scripts/make-release.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 912e706..555dd21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ *.swp debug debug-configuration.json -ipfix-rita-*.tgz +ipfix-rita.tgz diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index b9ff76b..db676ae 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -8,10 +8,9 @@ set -o errtrace set -o pipefail export IPFIX_RITA_VERSION="$(cat ../VERSION)" -IPFIX_RITA_VERSION_HYPHENATED="$(echo $IPFIX_RITA_VERSION | sed 's/\./-/g')" DOCKER_IMAGE_OUT="docker-images.tgz" -IPFIX_RITA_ARCHIVE="ipfix-rita-$IPFIX_RITA_VERSION_HYPHENATED" +IPFIX_RITA_ARCHIVE="ipfix-rita" IN_DEV_README="../README.md" IN_DEV_DOCS_DIR="../docs" @@ -82,7 +81,7 @@ done mkdir "$INSTALLER_BIN_DIR" # Construct the main script # Copy the shebang line in -head "$IN_DEV_MAIN_SCRIPT" >> "$INSTALLER_MAIN_SCRIPT" +head "$IN_DEV_MAIN_SCRIPT" >> "$INSTALLER_MAIN_SCRIPT" # Ensure the right versions of the images are ran echo "export IPFIX_RITA_VERSION=$IPFIX_RITA_VERSION" >> "$INSTALLER_MAIN_SCRIPT" @@ -206,6 +205,7 @@ if [ "\$SETUP_CONFIG" = "true" ]; then echo "In order to support this type of installation, you will need to" echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" echo "better if it is installed on a machine separate from RITA/ MongoDB." echo "" From 719dba9ffe691340b9fb06cff5a17222435e0c62 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 8 Jan 2019 12:00:07 -0700 Subject: [PATCH 02/19] Work in progress, I will still need to test this but good groundwork for breaking out IPFIX-RITA's installer to something nicer --- dev-scripts/make-release.sh | 284 +----------------- .../helpers/check_docker-compose.sh | 30 ++ install-scripts/helpers/check_docker.sh | 30 ++ install-scripts/install_docker.sh | 168 +++++++++++ install-scripts/install_ipfix-rita.sh | 282 +++++++++++++++++ 5 files changed, 518 insertions(+), 276 deletions(-) create mode 100755 install-scripts/helpers/check_docker-compose.sh create mode 100755 install-scripts/helpers/check_docker.sh create mode 100755 install-scripts/install_docker.sh create mode 100755 install-scripts/install_ipfix-rita.sh diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index db676ae..a3ee168 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -47,7 +47,7 @@ INSTALLER_ETC_DIR="$INSTALLER_PKG_DIR/etc" INSTALLER_COMPOSE_DIR="$INSTALLER_LIB_DIR/docker-compose" INSTALLER_MAIN_SCRIPT="$INSTALLER_BIN_DIR/ipfix-rita" -INSTALLER_INSTALL_SCRIPT="$INSTALLER_DIR/install-ipfix-rita.sh" +INSTALLER_SCRIPTS_DIR=$"../install-scripts" INSTALLER_TARBALL="./$IPFIX_RITA_ARCHIVE.tgz" @@ -61,6 +61,13 @@ if [ -f "$INSTALLER_TARBALL" ]; then rm "$INSTALLER_TARBALL" fi +# Insert the install scripts +# Start by adding the base install command +cp "$INSTALLER_SCRIPTS_DIR/install_ipfix-rita.sh" "$INSTALLER_DIR" +#Then add all the helper scirpts to the tarball and remove install-ipfix-rita.sh +cp -r "$INSTALLER_SCRIPTS_DIR" "$INSTALLER_DIR" +rm "$INSTALLER_DIR/install-scripts/install_ipfix-rita.sh" + # Copy in README cp "$IN_DEV_README" "$INSTALLER_README" @@ -96,281 +103,6 @@ docker save \ "quay.io/activecm/ipfix-rita-logstash:$IPFIX_RITA_VERSION" \ | gzip -c - > "$INSTALLER_DIR/$DOCKER_IMAGE_OUT" -# Insert the install script -cat << EOF > $INSTALLER_INSTALL_SCRIPT -#!/usr/bin/env bash -# Stop if there are any errors -set -e -# Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; - -if [[ \$EUID -ne 0 ]]; then - echo "This script must be run with administrator privileges." - exit 1 -fi - -# Ensure docker is functional -if [ ! -x "\$(command -v docker)" ]; then - echo "'docker' was not found in the PATH. Please install the latest" - echo "version of Docker using the official instructions for your OS." - exit 1 -fi - -DOCKER_VERSION="\$(docker -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_VERSION_MAJOR="\$(echo \$DOCKER_VERSION | cut -d' ' -f1)" -DOCKER_VERSION_MINOR="\$(echo \$DOCKER_VERSION | cut -d' ' -f2)" - -MIN_DOCKER_VERSION_MAJOR=17 -MIN_DOCKER_VERSION_MINOR=06 - -if [ "\$DOCKER_VERSION_MAJOR" -lt "\$MIN_DOCKER_VERSION_MAJOR" ] || -[ "\$DOCKER_VERSION_MAJOR" -eq "\$MIN_DOCKER_VERSION_MAJOR" -a "\$DOCKER_VERSION_MINOR" -lt "\$MIN_DOCKER_VERSION_MINOR" ]; then - echo "IPFIX_RITA requires Docker version \$MIN_DOCKER_VERSION_MAJOR.\$MIN_DOCKER_VERSION_MINOR+. Please upgrade to the latest" - echo "version of Docker using the official instructions for your OS." - exit 1 -fi - -# Ensure docker-compose is functional -if [ ! -x "\$(command -v docker-compose)" ]; then - echo "'docker-compose' was not found in the PATH. Please install the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi - -DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" -DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" - -MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 -MIN_DOCKER_COMPOSE_VERSION_MINOR=17 - -if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || -[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then - echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi - -# Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" -INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" -INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" - -echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load - -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" - -SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" -else - # TODO: set up migration - echo "Existing configuration found. Skipping..." - SETUP_CONFIG="false" -fi - -echo "Installing ipfix-rita in \$INSTALLATION_DIR" - -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" -fi - -mkdir -p "\$INSTALLATION_DIR" - -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" - -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" - -# set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf -fi - -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start - -if [ "\$SETUP_CONFIG" = "true" ]; then - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" - - echo "" - echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." - echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." - echo "In order to support this type of installation, you will need to" - echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." - echo "If needed, please do so, and restart MongoDB before continuing." - echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" - echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" - echo "better if it is installed on a machine separate from RITA/ MongoDB." - echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r - if [ -n "\$REPLY" ]; then - RITA_MONGO_URI="\$REPLY" - fi - - RITA_MONGO_AUTH="null" - - echo "" - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - done - - if [ "\$REPLY" = "2" ]; then - RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then - RITA_MONGO_AUTH="MONGODB-CR" - fi - - RITA_MONGO_TLS="false" - RITA_MONGO_TLS_CHECK_CERT="false" - RITA_MONGO_TLS_CERT_PATH="null" - echo "" - read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS="true" - RITA_MONGO_TLS_CHECK_CERT="true" - RITA_MONGO_TLS_CERT_PATH="null" - read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" - fi - - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then - read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS_CHECK_CERT="false" - fi - fi - fi - - echo "" - echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" - echo "where DBROOT consists of alphanumerics, underscores, and hyphens." - RITA_DATASET_DBROOT="IPFIX" - read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" - fi - - #unindent to ensure nothing breaks with awk - -awk -v db_root="\$RITA_DATASET_DBROOT" \\ --v mongo_uri="\$RITA_MONGO_URI" \\ --v mongo_auth="\$RITA_MONGO_AUTH" \\ --v mongo_tls_enable="\$RITA_MONGO_TLS" \\ --v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ --v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' -# flag is used to determine if we are in the right scope - -# Unset the flag if we see "abc:" or " abc:" on a line -# by itself if there are 2 or less preceding spaces -/^( )?[^ ]+:\$/{ - flag="" -} - -# Trigger the flag as we are entering the scope -/ RITA-MongoDB:/{ - flag=1 -} - -flag && NF && /ConnectionString:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_uri; - print val \$0; - next -} - -flag && NF && /AuthenticationMechanism:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_auth; - print val \$0; - next -} - -flag && NF && /Enable:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_enable; - print val \$0; - next -} - -flag && NF && /VerifyCertificate:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_cert_check; - print val \$0; - next -} - -flag && NF && /CAFile:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_ca_path; - print val \$0; - next -} - -flag && NF && /DBRoot:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=db_root; - print val \$0; - next -} - -1 -' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ -mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml - -echo "" -echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" -echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" -echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" -echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." -fi - -echo "" -echo "IPFIX-RITA will run at start up unless the system has been stopped." -echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." -echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." -echo "To view the system logs, run 'ipfix-rita logs -f'." -echo "" - -echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." - -ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita - -echo "" -echo "Starting IPFIX-RITA..." - -"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d - -echo "The IPFIX-RITA installer has finished." - -# Change back to the old directory at the end -cd \$_OLD_DIR; unset _OLD_DIR -EOF - -chmod +x "$INSTALLER_INSTALL_SCRIPT" - tar -C $TMP_DIR -czf $INSTALLER_TARBALL $IPFIX_RITA_ARCHIVE ################################################################################ diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh new file mode 100755 index 0000000..f4ce9b5 --- /dev/null +++ b/install-scripts/helpers/check_docker-compose.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Exit values +# 0 correct version installed +# 3 not installed +# 4 older than required minimum version +# 5 newer than required maximum version + +if [ ! -x "$(command -v docker-compose)" ]; then + exit 3 +fi + +MIN_VERSION_MAJOR=1 +MIN_VERSION_MINOR=16 +MAX_VERSION_MAJOR=1 +MAX_VERSION_MINOR=22 + +VERSION="$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" +VERSION_MINOR="$(echo $VERSION | cut -d' ' -f2)" + +if [ "$VERSION_MAJOR" -lt "$MIN_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MIN_VERSION_MAJOR" -a "$VERSION_MINOR" -lt "$MIN_VERSION_MINOR" ]; then + exit 4 +elif [ "$VERSION_MAJOR" -gt "$MAX_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MAX_VERSION_MAJOR" -a "$VERSION_MINOR" -gt "$MAX_VERSION_MINOR" ]; then + exit 5 +else + exit 0 +fi diff --git a/install-scripts/helpers/check_docker.sh b/install-scripts/helpers/check_docker.sh new file mode 100755 index 0000000..a2c3465 --- /dev/null +++ b/install-scripts/helpers/check_docker.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Exit values +# 0 correct version installed +# 3 not installed +# 4 older than required minimum version +# 5 newer than required maximum version + +if [ ! -x "$(command -v docker)" ]; then + exit 3 +fi + +MIN_VERSION_MAJOR=17 +MIN_VERSION_MINOR=05 +MAX_VERSION_MAJOR=18 +MAX_VERSION_MINOR=09 + +VERSION="$(docker -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" +VERSION_MINOR="$(echo $VERSION | cut -d' ' -f2)" + +if [ "$VERSION_MAJOR" -lt "$MIN_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MIN_VERSION_MAJOR" -a "$VERSION_MINOR" -lt "$MIN_VERSION_MINOR" ]; then + exit 4 +elif [ "$VERSION_MAJOR" -gt "$MAX_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MAX_VERSION_MAJOR" -a "$VERSION_MINOR" -gt "$MAX_VERSION_MINOR" ]; then + exit 5 +else + exit 0 +fi diff --git a/install-scripts/install_docker.sh b/install-scripts/install_docker.sh new file mode 100755 index 0000000..d33cdbc --- /dev/null +++ b/install-scripts/install_docker.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Change dir to script dir +pushd "$(dirname "$BASH_SOURCE[0]")" > /dev/null + +# Options and Usage +# ----------------------------------- +usage() { + scriptName=$(basename "$0") + echo -n "${scriptName} [OPTION]... +Install needed docker code to support IPFIX-RITA. +Options: + -g, --group-add Add the current user to the 'docker' group + -r, --replace-shell (Implies -g) When finished, replaces the current shell + so the current user can control docker immediately. + This will prevent any calling scripts from executing further. + -h, --help Display this help and exit +" +} + +askYN () { + TESTYN="" + while [ "$TESTYN" != 'Y' ] && [ "$TESTYN" != 'N' ] ; do + echo -n '?' >&2 + read TESTYN || : + case $TESTYN in + T*|t*|Y*|y*) TESTYN='Y' ;; + F*|f*|N*|n*) TESTYN='N' ;; + esac + done + + if [ "$TESTYN" = 'Y' ]; then + return 0 #True + else + return 1 #False + fi +} #End of askYN + +ADD_DOCKER_GROUP=false + +# Parse through command args to override values +while [[ $# -gt 0 ]]; do + case $1 in + -g|--group-add) + ADD_DOCKER_GROUP=true + ;; + -r|--replace-shell) + ADD_DOCKER_GROUP=true + REPLACE_SHELL=true + ;; + -h|--help) + usage >&2 + exit + ;; + *) + ;; + esac + shift +done + +helpers/check_docker.sh +# Store the exit code +DOCKER_CHECK=$? +if [ "$DOCKER_CHECK" -gt 3 ]; then + # This may overwrite a file maintained by a package. + echo "An unsupported version of Docker appears to already be installed. It will be replaced." +fi +if [ "$DOCKER_CHECK" -eq 0 ]; then + echo "Docker appears to already be installed. Skipping." +elif [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then + #This configuration file is used in both Redhat RHEL and Centos distributions, so we're running under RHEL/Centos 7.x + # https://docs.docker.com/engine/installation/linux/docker-ce/centos/ + + sudo yum makecache fast + + if rpm -q docker >/dev/null 2>&1 || rpm -q docker-common >/dev/null 2>&1 || rpm -q docker-selinux >/dev/null 2>&1 || rpm -q docker-engine >/dev/null 2>&1 ; then + echo -n "One or more of these packages are installed: docker, docker-common, docker-selinux, and/or docker-engine . The docker website encourages us to remove these before installing docker-ce. Would you like to remove these older packages (recommended: yes)" + if askYN ; then + sudo yum -y remove docker docker-common docker-selinux docker-engine + else + echo "You chose not to remove the older docker packages. The install may not succeed." + fi + fi + + sudo yum install -y yum-utils device-mapper-persistent-data lvm2 shadow-utils + + sudo yum-config-manager --enable extras + + if [ ! -f /etc/yum.repos.d/docker-ce.repo ]; then + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + fi + + sudo wget -q https://download.docker.com/linux/centos/gpg -O ~/DOCKER-GPG-KEY + sudo rpm --import ~/DOCKER-GPG-KEY + + sudo yum -y install docker-ce + + sudo systemctl start docker + sudo systemctl enable docker +elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then + ### Install Docker on Ubuntu ### + # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-repository + + echo "Installing Docker package repo..." + sudo apt-get update + sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + + echo "Installing latest Docker version..." + sudo apt-get update + sudo apt-get install -y docker-ce + sudo service docker start +else + echo "This system does not appear to be a Centos 7.x, RHEL 7.x, or Ubuntu Linux system. Unable to install docker." + exit 1 +fi + +helpers/check_docker-compose.sh +# Store the exit code +DOCKER_COMPOSE_CHECK=$? +if [ "$DOCKER_COMPOSE_CHECK" -gt 3 ]; then + # This may overwrite a file maintained by a package. + echo "An unsupported version of Docker-Compose appears to already be installed. It will be replaced." +fi +if [ "$DOCKER_COMPOSE_CHECK" -eq 0 ]; then + echo "Docker-Compose appears to already be installed. Skipping." +else + ### Install Docker-Compose ### + # https://docs.docker.com/compose/install/#install-compose + DOCKER_COMPOSE_VERSION="1.21.2" + + echo "Installing Docker-Compose v${DOCKER_COMPOSE_VERSION}..." + sudo curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o /usr/bin/docker-compose + sudo chmod +x /usr/bin/docker-compose +fi + +if [ "${ADD_DOCKER_GROUP}" = "true" ]; then + # Add current user to docker group + echo "Adding current user to docker group..." + #sudo groupadd docker + sudo usermod -aG docker $USER + + if [ "${REPLACE_SHELL}" = "true" ]; then + echo "Docker installation complete. You should have access to the 'docker' and 'docker-compose' commands immediately." + # Hack to activate the docker group on the current user without logging out. + # Downside is it completely replaces the shell and prevents calling scripts from continuing. + # https://superuser.com/a/853897 + exec sg docker newgrp `id -gn` + fi + + echo "You will need to login again for these changes to take effect." + echo "Docker installation complete. You should have access to the 'docker' and 'docker-compose' commands once you log out and back in." +else + echo "Docker installation complete. 'docker' and 'docker-compose' must be run using sudo or the root account." +fi + +# Change back to original directory +popd > /dev/null diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh new file mode 100755 index 0000000..4ffba60 --- /dev/null +++ b/install-scripts/install_ipfix-rita.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash + +askYN () { + TESTYN="" + while [ "$TESTYN" != 'Y' ] && [ "$TESTYN" != 'N' ] ; do + echo -n '?' >&2 + read TESTYN || : + case $TESTYN in + T*|t*|Y*|y*) TESTYN='Y' ;; + F*|f*|N*|n*) TESTYN='N' ;; + esac + done + + if [ "$TESTYN" = 'Y' ]; then + return 0 #True + else + return 1 #False + fi +} #End of askYN + +# Stop if there are any errors +set -e +# Change dir to script dir +_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; + +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run with administrator privileges." + exit 1 +fi + +# Install docker if needed +chmod +x "scripts/install_docker.sh" +sudo scripts/install_docker.sh +require_file /usr/bin/docker || fail "Docker does not appear to have been installed successfully" +if sudo docker run hello-world 2>/dev/null | grep -iq 'Hello from Docker!' ; then + status 'Docker appears to be working, continuing.' +else + echo 'Docker does not appear to be able to pull down a docker instance and run it successfully.' >&2 + echo 'The most likely explanation is that this system is not able to make outbound connections to the Internet.' >&2 + echo 'We recommend you fix this, but will proceed anyways in 20 seconds.' >&2 + sleep 20 +fi + +# Ensure docker-compose is functional +#TODO If we install docker automatically intall docker-compose automatically too +if [ ! -x "\$(command -v docker-compose)" ]; then + echo "'docker-compose' was not found in the PATH. Please install the latest" + echo "version of docker-compose using the official instructions for your OS." + exit 1 +fi + +DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" +DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" + +MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 +MIN_DOCKER_COMPOSE_VERSION_MINOR=17 + +if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || +[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then + echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" + echo "version of docker-compose using the official instructions for your OS." + exit 1 +fi +#end of TODO's + +# Set by make-release +INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" +INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" +INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" +DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + +echo "Loading IPFIX-RITA Docker images... This may take a few minutes." +gzip -d -c \${DOCKER_IMAGES} | docker load + +echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + +SETUP_CONFIG="true" +if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +else + # TODO: set up migration + echo "Existing configuration found. Skipping..." + SETUP_CONFIG="false" +fi + +echo "Installing ipfix-rita in \$INSTALLATION_DIR" + +if [ -d "\$INSTALLATION_DIR" ]; then + rm -rf "\$INSTALLATION_DIR" +fi + +mkdir -p "\$INSTALLATION_DIR" + +cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" +chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + +cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + +# set receive buffer size for logstash collector +RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" +if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +fi + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + +if [ "\$SETUP_CONFIG" = "true" ]; then + IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + + echo "" + echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." + echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." + echo "In order to support this type of installation, you will need to" + echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." + echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" + echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" + echo "better if it is installed on a machine separate from RITA/ MongoDB." + echo "" + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + if [ -n "\$REPLY" ]; then + RITA_MONGO_URI="\$REPLY" + fi + + RITA_MONGO_AUTH="null" + + echo "" + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + + while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + done + + if [ "\$REPLY" = "2" ]; then + RITA_MONGO_AUTH="SCRAM-SHA-1" + elif [ "\$REPLY" = "3" ]; then + RITA_MONGO_AUTH="MONGODB-CR" + fi + + RITA_MONGO_TLS="false" + RITA_MONGO_TLS_CHECK_CERT="false" + RITA_MONGO_TLS_CERT_PATH="null" + echo "" + read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS="true" + RITA_MONGO_TLS_CHECK_CERT="true" + RITA_MONGO_TLS_CERT_PATH="null" + read -p "Would you like to provide a certificate authority? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + read -p "CA Path: " + RITA_MONGO_TLS_CERT_PATH="\$REPLY" + fi + + if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + read -p "Would you like to disable certificate checks? [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS_CHECK_CERT="false" + fi + fi + fi + + echo "" + echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" + echo "where DBROOT consists of alphanumerics, underscores, and hyphens." + RITA_DATASET_DBROOT="IPFIX" + read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r + if [ -n "\$REPLY" ]; then + RITA_DATASET_DBROOT="\$REPLY" + fi + + #unindent to ensure nothing breaks with awk + +awk -v db_root="\$RITA_DATASET_DBROOT" \\ +-v mongo_uri="\$RITA_MONGO_URI" \\ +-v mongo_auth="\$RITA_MONGO_AUTH" \\ +-v mongo_tls_enable="\$RITA_MONGO_TLS" \\ +-v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ +-v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' +# flag is used to determine if we are in the right scope + +# Unset the flag if we see "abc:" or " abc:" on a line +# by itself if there are 2 or less preceding spaces +/^( )?[^ ]+:\$/{ + flag="" +} + +# Trigger the flag as we are entering the scope +/ RITA-MongoDB:/{ + flag=1 +} + +flag && NF && /ConnectionString:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_uri; + print val \$0; + next +} + +flag && NF && /AuthenticationMechanism:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_auth; + print val \$0; + next +} + +flag && NF && /Enable:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_enable; + print val \$0; + next +} + +flag && NF && /VerifyCertificate:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_cert_check; + print val \$0; + next +} + +flag && NF && /CAFile:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_ca_path; + print val \$0; + next +} + +flag && NF && /DBRoot:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=db_root; + print val \$0; + next +} + +1 +' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml + +echo "" +echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" +echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" +echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" +echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." +fi + +echo "" +echo "IPFIX-RITA will run at start up unless the system has been stopped." +echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." +echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." +echo "To view the system logs, run 'ipfix-rita logs -f'." +echo "" + +echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." + +ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita + +echo "" +echo "Starting IPFIX-RITA..." + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d + +echo "The IPFIX-RITA installer has finished." + +# Change back to the old directory at the end +cd \$_OLD_DIR; unset _OLD_DIR From 4f736fb628d1a8da3bdd0d24fd6b0d458c1c54a2 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Sat, 12 Jan 2019 11:10:28 -0700 Subject: [PATCH 03/19] Moving code to different sections to enable a non-interactive version of the installer. Still in the testing/debugging stages --- .../helpers/check_docker-compose.sh | 2 +- install-scripts/install_ipfix-rita.sh | 489 ++++++++++++------ 2 files changed, 340 insertions(+), 151 deletions(-) diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh index f4ce9b5..b7570fe 100755 --- a/install-scripts/helpers/check_docker-compose.sh +++ b/install-scripts/helpers/check_docker-compose.sh @@ -11,7 +11,7 @@ if [ ! -x "$(command -v docker-compose)" ]; then fi MIN_VERSION_MAJOR=1 -MIN_VERSION_MINOR=16 +MIN_VERSION_MINOR=17 MAX_VERSION_MAJOR=1 MAX_VERSION_MINOR=22 diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 4ffba60..ca59759 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -18,169 +18,262 @@ askYN () { fi } #End of askYN -# Stop if there are any errors -set -e -# Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; +check_remote_mongo() { + hostip=$1 + if [`echo 'show dbs' | mongo --host $hostip | grep '^bye'` = 'bye']; then + return 0 #True + else + return 1 #False + fi +} #End of check_remote_mongo + +can_ssh () { + #Test that we can reach the target system over ssh. + success_code=1 + if [ "$1" = "127.0.0.1" ]; then + success_code=0 + elif [ -n "$1" ]; then + token="$RANDOM.$RANDOM" + echo "Attempting to verify that we can ssh to $@ - you may need to provide a password to access this system." >&2 + ssh_out=`ssh "$@" '/bin/echo '"$token"` + if [ "$token" = "$ssh_out" ]; then + #echo "successfully connected to $@" + success_code=0 + #else + #echo "cannot connect to $@" + fi + else + echo "Please supply an ssh target as a command line parameter to can_ssh" + fi -if [[ \$EUID -ne 0 ]]; then - echo "This script must be run with administrator privileges." - exit 1 -fi + return $success_code +} #End of can_ssh -# Install docker if needed -chmod +x "scripts/install_docker.sh" -sudo scripts/install_docker.sh -require_file /usr/bin/docker || fail "Docker does not appear to have been installed successfully" -if sudo docker run hello-world 2>/dev/null | grep -iq 'Hello from Docker!' ; then - status 'Docker appears to be working, continuing.' -else - echo 'Docker does not appear to be able to pull down a docker instance and run it successfully.' >&2 - echo 'The most likely explanation is that this system is not able to make outbound connections to the Internet.' >&2 - echo 'We recommend you fix this, but will proceed anyways in 20 seconds.' >&2 - sleep 20 -fi +do_system_tests () { + while [ -n "$1" ]; do + if [ "$1" = "check_sse4_2" ]; then + status "Hardware tests" + require_file /proc/cpuinfo || fail "Missing /proc/cpuinfo - is this a Linux system?" + fi + shift + done -# Ensure docker-compose is functional -#TODO If we install docker automatically intall docker-compose automatically too -if [ ! -x "\$(command -v docker-compose)" ]; then - echo "'docker-compose' was not found in the PATH. Please install the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi + for one_dir in $HOME / /usr ; do + #"tr" in next command removes the linefeed at the end of the number of free megabytes (and any other whitespace) + [ `df -h $one_dir --output=avail --block-size=1048576 | grep -v '^Avail' | tr -dc '[0-9]'` -gt 5000 ] || fail "${one_dir} has less than 5GB free space" + done -DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" -DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" + status "Installed software tests" + if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then + #We have apt-get , good. + sudo apt-get -qq update > /dev/null 2>&1 + inst_retcode=100000 + while [ "$inst_retcode" -gt 0 ]; do + sudo apt-get -y install gdb git wget curl make realpath lsb-release rsync tar + inst_retcode=$? + if [ "$inst_retcode" -gt 0 ]; then + echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" + sleep 60 + fi + done + #TODO test IPFIX-RITA on Contos/Red Hat System + #elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then + # if [ ! -x /bin/yum-config-manager ]; then + # sudo yum -y install yum-utils + # fi + #We have yum, good. + # sudo yum -q makecache > /dev/null 2>&1 + # #FIXME - put in place a similar loop like above for apt-get + # sudo yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar + else + fail "(apt-get and dpkg-query) is installed on this system"#nor (yum, rpm, and yum-config-manager) is installed on the system" + fi + require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" -MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 -MIN_DOCKER_COMPOSE_VERSION_MINOR=17 + if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then + #echo "Centos or Redhat 7 installation detected, good." + fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then + echo "Ubuntu Linux installation detected, good." + else + fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + fi -if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || -[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then - echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi -#end of TODO's + return 0 +} #End of do_system_tests + +validated_ssh_target () { + #Return an ssh target that we've confirmed we can reach. "$2" is the initial target to try, and may be replaced. + #The potential target - both as supplied as parameter 2 and as returned at the end - may be blank, indicating "do not install this" + target_type="$1" + potential_target="$2" + + echo 'About to execute commands on '"$potential_target"' . You may be prompted one or more times for the password for this system.' >&2 + echo 'Note: if you are using an ssh client other than openssh and have not set up ssh key access to this system, you may be prompted for your password multiple times.' >&2 + while [ -n "$potential_target" ] && ! can_ssh "$potential_target" ; do + echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to install on this system. Just press enter if you do not wish to set up IPFIX logging. Press Ctrl-C to exit." >&2 + read potential_target <&2 + if [ -n "$potential_target" -a "$potential_target" != '127.0.0.1' ]; then + echo "Do you access that system with a username different than ${USER}? If so, enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read potential_user <&2 + if [ -n "$potential_user" ]; then + potential_target="${potential_user}@${potential_target}" + fi + fi + done -# Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" -INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" -INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + echo "$potential_target" +} #End of validated_ssh_tartget + +get_line() { + rita_file=$1 + search_key=$2 + + #get the line that matches the search key that isn't commented out + line=`cat $rita_file | grep $search_key | grep -v "#"` + +} #End of get line + +get_rita_data_interactive() { + IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + + echo "" + echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." + echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." + echo "In order to support this type of installation, you will need to" + echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." + echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" + echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" + echo "better if it is installed on a machine separate from RITA/ MongoDB." + echo "" + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + read RITA_MONGO_URI <&2 + if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then + echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 + if askYN ; then + echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read rita_user <&2 + if [ -n "$rita_user" ]; then + rita_system="${rita_user}@${rita_system}" + fi + fi + rita_system=`validated_ssh_target rita "$rita_system"` + else + rita_system="127.0.0.1" + fi -echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load + RITA_MONGO_AUTH="null" + #instead of prompting get the RITA address and split from there to fill in our + # config file + echo "" + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + + while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + done -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + if [ "\$REPLY" = "2" ]; then + RITA_MONGO_AUTH="SCRAM-SHA-1" + elif [ "\$REPLY" = "3" ]; then + RITA_MONGO_AUTH="MONGODB-CR" + fi -SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" -else - # TODO: set up migration - echo "Existing configuration found. Skipping..." - SETUP_CONFIG="false" -fi + RITA_MONGO_TLS="false" + RITA_MONGO_TLS_CHECK_CERT="false" + RITA_MONGO_TLS_CERT_PATH="null" + echo "" + read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS="true" + RITA_MONGO_TLS_CHECK_CERT="true" + RITA_MONGO_TLS_CERT_PATH="null" + read -p "Would you like to provide a certificate authority? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + read -p "CA Path: " + RITA_MONGO_TLS_CERT_PATH="\$REPLY" + fi + + if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + read -p "Would you like to disable certificate checks? [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS_CHECK_CERT="false" + fi + fi + fi -echo "Installing ipfix-rita in \$INSTALLATION_DIR" + echo "" + echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" + echo "where DBROOT consists of alphanumerics, underscores, and hyphens." + RITA_DATASET_DBROOT="IPFIX" + read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r + if [ -n "\$REPLY" ]; then + RITA_DATASET_DBROOT="\$REPLY" + fi -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" -fi + write_converter_conf $RITA_DATASET_DBROOT $RITA_MONGO_URI $RITA_MONGO_AUTH $RITA_MONGO_TLS $RITA_MONGO_TLS_CHECK_CERT $RITA_MONGO_TLS_CERT_PATH +} #End of get_rita_data_interactive + +get_rita_data_noninteractive() { + rita_system=$1 + if [ -n "$rita_system" -a "$rita_system" != '127.0.0.1' ]; then + echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 + if askYN ; then + echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read rita_user <&2 + if [ -n "$rita_user" ]; then + rita_system="${rita_user}@${rita_system}" + fi + fi + rita_system=`validated_ssh_target rita "$rita_system"` + else + rita_system="127.0.0.1" + fi -mkdir -p "\$INSTALLATION_DIR" + if [ "$rita_system" != '127.0.0.1' ]; then + scp $rita_system:/etc/rita/config.yaml . + rita_conf="./config.yaml" + delete_conf=true + else + #do something great here + rita_conf="/etc/rita/config.yaml" + fi -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + db_root=${rita_system#*@} + mongo_uri="mongodb://$db_root:$MONGO_PORT" + mongo_auth=`get_line $rita_conf "AuthenticationMechanism: "` + mongo_tls_enable=`get_line $rita_conf "Enable: "` + mongo_tls_cert_check=`get_line $rita_conf "VerifyCertificate: "` + mongo_tls_ca_path=`get_line $rita_conf "CAFile: "` -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + #If the config file was copied from a remote server, delete that file + if [ "$delete_conf" != "true" ]; then + rm "./config.yaml" + fi -# set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf -fi + #If the config specified a CA file, let's copy that + if [[ $mongo_tls_ca_path != *"null" ]]; then + ca_file=${mongo_tls_ca_path#"CAFile: "} + scp $rita_system:$ca_file $ca_file + fi -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + write_converter_conf $db_root $mongo_uri $mongo_auth $mongo_tls_enable $mongo_tls_cert_check $mongo_tls_ca_path +} #End of get_rita_data_noninteractive -if [ "\$SETUP_CONFIG" = "true" ]; then - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" - - echo "" - echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." - echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." - echo "In order to support this type of installation, you will need to" - echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." - echo "If needed, please do so, and restart MongoDB before continuing." - echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" - echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" - echo "better if it is installed on a machine separate from RITA/ MongoDB." - echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r - if [ -n "\$REPLY" ]; then - RITA_MONGO_URI="\$REPLY" - fi - - RITA_MONGO_AUTH="null" - - echo "" - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - done - - if [ "\$REPLY" = "2" ]; then - RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then - RITA_MONGO_AUTH="MONGODB-CR" - fi - - RITA_MONGO_TLS="false" - RITA_MONGO_TLS_CHECK_CERT="false" - RITA_MONGO_TLS_CERT_PATH="null" - echo "" - read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS="true" - RITA_MONGO_TLS_CHECK_CERT="true" - RITA_MONGO_TLS_CERT_PATH="null" - read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" - fi - - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then - read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS_CHECK_CERT="false" - fi - fi - fi - - echo "" - echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" - echo "where DBROOT consists of alphanumerics, underscores, and hyphens." - RITA_DATASET_DBROOT="IPFIX" - read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" - fi - - #unindent to ensure nothing breaks with awk +write_converter_conf() { + RITA_DATASET_DBROOT=$1 + RITA_MONGO_URI=$2 + RITA_MONGO_AUTH=$3 + RITA_MONGO_TLS=$4 + RITA_MONGO_TLS_CHECK_CERT=$5 + RITA_MONGO_TLS_CERT_PATH=$6 awk -v db_root="\$RITA_DATASET_DBROOT" \\ -v mongo_uri="\$RITA_MONGO_URI" \\ @@ -252,12 +345,108 @@ flag && NF && /DBRoot:/{ 1 ' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml +} #End of write_converter_conf -echo "" -echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" -echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" -echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" -echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." +# Stop if there are any errors +set -e +# Change dir to script dir +_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; + +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run with administrator privileges." + exit 1 +fi + +# Parse through command args to override values +echo "Loading command line params" >&2 +if [ "$1" = 'help' -o "$1" = '--help' ]; then + usage_text + exit 1 +fi + +while [ -n "$1" ]; do + case "$1" in + rita|Rita|RITA) + if [ -n "$2" ] && ! echo "$2" | egrep -iq '(^rita$|^aih$)' ; then + rita_system="$2" + shift + else + rita_system='127.0.0.1' + fi + ;; + *) + echo "Unrecognized parameter $1" >&2 + usage_text + exit 1 + ;; + esac + shift +done + +#TODO check system requirements + +# Install docker if needed +chmod +x "scripts/install_docker.sh" +sudo scripts/install_docker.sh + +# Set by make-release +INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" +INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" +INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" +DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + +echo "Loading IPFIX-RITA Docker images... This may take a few minutes." +gzip -d -c \${DOCKER_IMAGES} | docker load + +echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + +SETUP_CONFIG="true" +if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +else + # TODO: set up migration + echo "Existing configuration found. Skipping..." + SETUP_CONFIG="false" +fi + +echo "Installing ipfix-rita in \$INSTALLATION_DIR" + +if [ -d "\$INSTALLATION_DIR" ]; then + rm -rf "\$INSTALLATION_DIR" +fi + +mkdir -p "\$INSTALLATION_DIR" + +cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" +chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + +cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + +# set receive buffer size for logstash collector +RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" +if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +fi + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + +if [ "\$SETUP_CONFIG" = "true" ]; then + #From here out we want to move to a interactive/active install command + if [ -n "$rita_system" ]; then + get_rita_data_noninteractive $rita_system + else + get_rita_data_interactive + fi + +#We have written to config + echo "" + echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" + echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" + echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" + echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." fi echo "" From 719f578f6fa3deb67c89da30b748382b06a72563 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 15 Jan 2019 16:34:03 -0700 Subject: [PATCH 04/19] Made some modifications to the install script, still needs debugging and additional work --- install-scripts/install_ipfix-rita.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index ca59759..045940b 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -27,6 +27,8 @@ check_remote_mongo() { fi } #End of check_remote_mongo + + can_ssh () { #Test that we can reach the target system over ssh. success_code=1 @@ -49,6 +51,21 @@ can_ssh () { return $success_code } #End of can_ssh + + +fail () { + echo "$* ." >&2 + echo "This is normally an unrecoverable problem, and we recommend fixing the problem and restarting the script. Please contact technical support for help in resolving the issue. If you feel the script should continue, enter Y and the script will attempt to finish. Entering N will cause this script to exit." >&2 + if askYN ; then + echo "Script will continue at user request. This may not result in a working configuration." >&2 + sleep 5 + else + exit 1 + fi +} #end of fail + + + do_system_tests () { while [ -n "$1" ]; do if [ "$1" = "check_sse4_2" ]; then @@ -91,8 +108,7 @@ do_system_tests () { require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then - #echo "Centos or Redhat 7 installation detected, good." - fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + echo "Centos or Redhat 7 installation detected, good." elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then echo "Ubuntu Linux installation detected, good." else From 1373dc5e573d5fb6c3ac31722730aeed1b67e6da Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Wed, 16 Jan 2019 01:01:31 -0700 Subject: [PATCH 05/19] Adding changes to make the install interactive functional. Need to add mongo check and test/debug non-interactive commands --- dev-scripts/make-release.sh | 3 + .../helpers/check_docker-compose.sh | 2 +- install-scripts/install_ipfix-rita.sh | 110 +++++++++--------- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index a3ee168..7f50584 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -64,6 +64,9 @@ fi # Insert the install scripts # Start by adding the base install command cp "$INSTALLER_SCRIPTS_DIR/install_ipfix-rita.sh" "$INSTALLER_DIR" +sed -i "s|INSTALLATION_DIR=\"REPLACE_WITH_INSTALL_DIR\"|INSTALLATION_DIR=\"$INSTALLATION_DIR\"|g" $INSTALLER_DIR/install_ipfix-rita.sh +sed -i "s|INSTALLATION_ETC_DIR=\"REPLACE_WITH_ETC_DIR\"|INSTALLATION_ETC_DIR=\"$INSTALLATION_ETC_DIR\"|g" $INSTALLER_DIR/install_ipfix-rita.sh +sed -i "s|DOCKER_IMAGES=\"./REPLACE_WITH_TARBALL\"|DOCKER_IMAGES=\"$DOCKER_IMAGE_OUT\"|g" $INSTALLER_DIR/install_ipfix-rita.sh #Then add all the helper scirpts to the tarball and remove install-ipfix-rita.sh cp -r "$INSTALLER_SCRIPTS_DIR" "$INSTALLER_DIR" rm "$INSTALLER_DIR/install-scripts/install_ipfix-rita.sh" diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh index b7570fe..e7cc6ae 100755 --- a/install-scripts/helpers/check_docker-compose.sh +++ b/install-scripts/helpers/check_docker-compose.sh @@ -13,7 +13,7 @@ fi MIN_VERSION_MAJOR=1 MIN_VERSION_MINOR=17 MAX_VERSION_MAJOR=1 -MAX_VERSION_MINOR=22 +MAX_VERSION_MINOR=23 VERSION="$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 045940b..21632c3 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -103,7 +103,7 @@ do_system_tests () { # #FIXME - put in place a similar loop like above for apt-get # sudo yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar else - fail "(apt-get and dpkg-query) is installed on this system"#nor (yum, rpm, and yum-config-manager) is installed on the system" + fail "(apt-get and dpkg-query) is installed on this system" fi require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" @@ -127,7 +127,7 @@ validated_ssh_target () { echo 'About to execute commands on '"$potential_target"' . You may be prompted one or more times for the password for this system.' >&2 echo 'Note: if you are using an ssh client other than openssh and have not set up ssh key access to this system, you may be prompted for your password multiple times.' >&2 while [ -n "$potential_target" ] && ! can_ssh "$potential_target" ; do - echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to install on this system. Just press enter if you do not wish to set up IPFIX logging. Press Ctrl-C to exit." >&2 + echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to run on this system. Press Ctrl-C to exit." >&2 read potential_target <&2 if [ -n "$potential_target" -a "$potential_target" != '127.0.0.1' ]; then echo "Do you access that system with a username different than ${USER}? If so, enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 @@ -151,8 +151,8 @@ get_line() { } #End of get line get_rita_data_interactive() { - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + IPFIX_RITA_NETWORK_GATEWAY=$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://$IPFIX_RITA_NETWORK_GATEWAY:27017" echo "" echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." @@ -164,7 +164,7 @@ get_rita_data_interactive() { echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" echo "better if it is installed on a machine separate from RITA/ MongoDB." echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [$RITA_MONGO_URI]: " -r read RITA_MONGO_URI <&2 if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 @@ -180,6 +180,8 @@ get_rita_data_interactive() { rita_system="127.0.0.1" fi + #TODO check if the mongo server is available + RITA_MONGO_AUTH="null" #instead of prompting get the RITA address and split from there to fill in our # config file @@ -189,16 +191,16 @@ get_rita_data_interactive() { echo "2) SCRAM-SHA-1" echo "3) MONGODB-CR" - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + while read && [[ ! ( "$REPLY" =~ ^[123]$ || -z "$REPLY" ) ]]; do echo "Which authentication scheme should be used to contact the database if any? [None]" echo "1) None" echo "2) SCRAM-SHA-1" echo "3) MONGODB-CR" done - if [ "\$REPLY" = "2" ]; then + if [ "$REPLY" = "2" ]; then RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then + elif [ "$REPLY" = "3" ]; then RITA_MONGO_AUTH="MONGODB-CR" fi @@ -207,19 +209,19 @@ get_rita_data_interactive() { RITA_MONGO_TLS_CERT_PATH="null" echo "" read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS="true" RITA_MONGO_TLS_CHECK_CERT="true" RITA_MONGO_TLS_CERT_PATH="null" read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" + RITA_MONGO_TLS_CERT_PATH="$REPLY" fi - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + if [ "$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS_CHECK_CERT="false" fi fi @@ -230,8 +232,8 @@ get_rita_data_interactive() { echo "where DBROOT consists of alphanumerics, underscores, and hyphens." RITA_DATASET_DBROOT="IPFIX" read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" + if [ -n "$REPLY" ]; then + RITA_DATASET_DBROOT="$REPLY" fi write_converter_conf $RITA_DATASET_DBROOT $RITA_MONGO_URI $RITA_MONGO_AUTH $RITA_MONGO_TLS $RITA_MONGO_TLS_CHECK_CERT $RITA_MONGO_TLS_CERT_PATH @@ -291,12 +293,12 @@ write_converter_conf() { RITA_MONGO_TLS_CHECK_CERT=$5 RITA_MONGO_TLS_CERT_PATH=$6 -awk -v db_root="\$RITA_DATASET_DBROOT" \\ --v mongo_uri="\$RITA_MONGO_URI" \\ --v mongo_auth="\$RITA_MONGO_AUTH" \\ --v mongo_tls_enable="\$RITA_MONGO_TLS" \\ --v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ --v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' +awk -v db_root="$RITA_DATASET_DBROOT" \ +-v mongo_uri="$RITA_MONGO_URI" \ +-v mongo_auth="$RITA_MONGO_AUTH" \ +-v mongo_tls_enable="$RITA_MONGO_TLS" \ +-v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ +-v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' # flag is used to determine if we are in the right scope # Unset the flag if we see "abc:" or " abc:" on a line @@ -359,16 +361,16 @@ flag && NF && /DBRoot:/{ } 1 -' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ -mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml +' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors set -e # Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; +_OLD_DIR=$(pwd); cd "$(dirname "$BASH_SOURCE[0]")"; -if [[ \$EUID -ne 0 ]]; then +if [[ $EUID -ne 0 ]]; then echo "This script must be run with administrator privileges." exit 1 fi @@ -402,54 +404,54 @@ done #TODO check system requirements # Install docker if needed -chmod +x "scripts/install_docker.sh" -sudo scripts/install_docker.sh +chmod +x "install-scripts/install_docker.sh" +sudo install-scripts/install_docker.sh # Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_DIR="REPLACE_WITH_INSTALL_DIR" INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" +INSTALLATION_ETC_DIR="REPLACE_WITH_ETC_DIR" +DOCKER_IMAGES="./REPLACE_WITH_TARBALL" echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load +gzip -d -c ${DOCKER_IMAGES} | docker load -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" +echo "Installing configuration files to $INSTALLATION_ETC_DIR" SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +if [ ! -d "$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "$INSTALLATION_ETC_DIR" else # TODO: set up migration echo "Existing configuration found. Skipping..." SETUP_CONFIG="false" fi -echo "Installing ipfix-rita in \$INSTALLATION_DIR" +echo "Installing ipfix-rita in $INSTALLATION_DIR" -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" +if [ -d "$INSTALLATION_DIR" ]; then + rm -rf "$INSTALLATION_DIR" fi -mkdir -p "\$INSTALLATION_DIR" +mkdir -p "$INSTALLATION_DIR" -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" +cp -r ./pkg/bin "$INSTALLATION_BIN_DIR" +chmod +x "$INSTALLATION_BIN_DIR/ipfix-rita" -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" +cp -r ./pkg/lib "$INSTALLATION_LIB_DIR" # set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +RECV_BUFF_SIZE=$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="$((1024*1024*64))" +if [ "$RECV_BUFF_SIZE" -lt "$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf fi -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start +"$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start -if [ "\$SETUP_CONFIG" = "true" ]; then +if [ "$SETUP_CONFIG" = "true" ]; then #From here out we want to move to a interactive/active install command if [ -n "$rita_system" ]; then get_rita_data_noninteractive $rita_system @@ -459,10 +461,10 @@ if [ "\$SETUP_CONFIG" = "true" ]; then #We have written to config echo "" - echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" + echo "Your settings have been saved to $INSTALLATION_ETC_DIR/converter/converter.yaml" echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" - echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." + echo "in $INSTALLATION_ETC_DIR/converter/converter.yaml." fi echo "" @@ -472,16 +474,16 @@ echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." echo "To view the system logs, run 'ipfix-rita logs -f'." echo "" -echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." +echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to $INSTALLATION_BIN_DIR/ipfix-rita." -ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita +ln -fs "$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita echo "" echo "Starting IPFIX-RITA..." -"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d +"$INSTALLATION_BIN_DIR/ipfix-rita" up -d echo "The IPFIX-RITA installer has finished." # Change back to the old directory at the end -cd \$_OLD_DIR; unset _OLD_DIR +cd $_OLD_DIR; unset _OLD_DIR From 64d10389da5987dc3d0a12cdbb31b3d3f51bbba7 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 5 Feb 2019 20:28:28 -0700 Subject: [PATCH 06/19] Adding changes to install script, works fine for non-interactive. Still having awk(ward) problems --- install-scripts/install_ipfix-rita.sh | 86 ++++++++++++++++----------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 21632c3..15e3749 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -118,6 +118,16 @@ do_system_tests () { return 0 } #End of do_system_tests +usage_text () { + cat >&2 < $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \ mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors set -e # Change dir to script dir +echo "tyring to cd to $(dirname "$BASH_SOURCE[0]")" _OLD_DIR=$(pwd); cd "$(dirname "$BASH_SOURCE[0]")"; if [[ $EUID -ne 0 ]]; then From cefbb03439e418c7c6b749dcfa77b00fb4a68ba2 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Wed, 6 Feb 2019 11:34:12 -0700 Subject: [PATCH 07/19] Finished the noninteractive flag debugging --- install-scripts/install_ipfix-rita.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 15e3749..a88a933 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -306,17 +306,17 @@ write_converter_conf() { RITA_MONGO_TLS_CHECK_CERT=$5 RITA_MONGO_TLS_CERT_PATH=$6 - awk -v db_root="$RITA_DATASET_DBROOT" \ - -v mongo_uri="$RITA_MONGO_URI" \ - -v mongo_auth="$RITA_MONGO_AUTH" \ - -v mongo_tls_enable="$RITA_MONGO_TLS" \ - -v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ - -v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' +awk -v db_root="$RITA_DATASET_DBROOT" \ +-v mongo_uri="$RITA_MONGO_URI" \ +-v mongo_auth="$RITA_MONGO_AUTH" \ +-v mongo_tls_enable="$RITA_MONGO_TLS" \ +-v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ +-v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' # flag is used to determine if we are in the right scope # Unset the flag if we see "abc:" or " abc:" on a line # by itself if there are 2 or less preceding spaces -/^( )?[^ ]+:\$/{ +/^( )?[^ ]+:$/{ flag="" } @@ -375,7 +375,7 @@ flag && NF && /DBRoot:/{ 1 ' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \ -mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml + mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors From 5db2f7fdf83654c4e7a8b67dabfc5f1e38c540e1 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Fri, 4 Jan 2019 13:37:04 -0700 Subject: [PATCH 08/19] Removing version from build script, and adding MongoDB restart command (might need to rearrange so it pops more) --- .gitignore | 2 +- dev-scripts/make-release.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 912e706..555dd21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ *.swp debug debug-configuration.json -ipfix-rita-*.tgz +ipfix-rita.tgz diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index 47069ec..6f414f0 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -8,10 +8,9 @@ set -o errtrace set -o pipefail export IPFIX_RITA_VERSION="$(cat ../VERSION)" -IPFIX_RITA_VERSION_HYPHENATED="$(echo $IPFIX_RITA_VERSION | sed 's/\./-/g')" DOCKER_IMAGE_OUT="docker-images.tgz" -IPFIX_RITA_ARCHIVE="ipfix-rita-$IPFIX_RITA_VERSION_HYPHENATED" +IPFIX_RITA_ARCHIVE="ipfix-rita" IN_DEV_README="../README.md" IN_DEV_DOCS_DIR="../docs" @@ -206,6 +205,7 @@ if [ "\$SETUP_CONFIG" = "true" ]; then echo "In order to support this type of installation, you will need to" echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" echo "better if it is installed on a machine separate from RITA/ MongoDB." echo "" From ea01e150852b70108c50bf1d44a8ec15536436e8 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 8 Jan 2019 12:00:07 -0700 Subject: [PATCH 09/19] Work in progress, I will still need to test this but good groundwork for breaking out IPFIX-RITA's installer to something nicer --- dev-scripts/make-release.sh | 284 +----------------- .../helpers/check_docker-compose.sh | 30 ++ install-scripts/helpers/check_docker.sh | 30 ++ install-scripts/install_docker.sh | 168 +++++++++++ install-scripts/install_ipfix-rita.sh | 282 +++++++++++++++++ 5 files changed, 518 insertions(+), 276 deletions(-) create mode 100755 install-scripts/helpers/check_docker-compose.sh create mode 100755 install-scripts/helpers/check_docker.sh create mode 100755 install-scripts/install_docker.sh create mode 100755 install-scripts/install_ipfix-rita.sh diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index 6f414f0..e017dcc 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -47,7 +47,7 @@ INSTALLER_ETC_DIR="$INSTALLER_PKG_DIR/etc" INSTALLER_COMPOSE_DIR="$INSTALLER_LIB_DIR/docker-compose" INSTALLER_MAIN_SCRIPT="$INSTALLER_BIN_DIR/ipfix-rita" -INSTALLER_INSTALL_SCRIPT="$INSTALLER_DIR/install-ipfix-rita.sh" +INSTALLER_SCRIPTS_DIR=$"../install-scripts" INSTALLER_TARBALL="./$IPFIX_RITA_ARCHIVE.tgz" @@ -61,6 +61,13 @@ if [ -f "$INSTALLER_TARBALL" ]; then rm "$INSTALLER_TARBALL" fi +# Insert the install scripts +# Start by adding the base install command +cp "$INSTALLER_SCRIPTS_DIR/install_ipfix-rita.sh" "$INSTALLER_DIR" +#Then add all the helper scirpts to the tarball and remove install-ipfix-rita.sh +cp -r "$INSTALLER_SCRIPTS_DIR" "$INSTALLER_DIR" +rm "$INSTALLER_DIR/install-scripts/install_ipfix-rita.sh" + # Copy in README cp "$IN_DEV_README" "$INSTALLER_README" @@ -96,281 +103,6 @@ docker save \ "quay.io/activecm/ipfix-rita-logstash:$IPFIX_RITA_VERSION" \ | gzip -c - > "$INSTALLER_DIR/$DOCKER_IMAGE_OUT" -# Insert the install script -cat << EOF > $INSTALLER_INSTALL_SCRIPT -#!/usr/bin/env bash -# Stop if there are any errors -set -e -# Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; - -if [[ \$EUID -ne 0 ]]; then - echo "This script must be run with administrator privileges." - exit 1 -fi - -# Ensure docker is functional -if [ ! -x "\$(command -v docker)" ]; then - echo "'docker' was not found in the PATH. Please install the latest" - echo "version of Docker using the official instructions for your OS." - exit 1 -fi - -DOCKER_VERSION="\$(docker -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_VERSION_MAJOR="\$(echo \$DOCKER_VERSION | cut -d' ' -f1)" -DOCKER_VERSION_MINOR="\$(echo \$DOCKER_VERSION | cut -d' ' -f2)" - -MIN_DOCKER_VERSION_MAJOR=17 -MIN_DOCKER_VERSION_MINOR=06 - -if [ "\$DOCKER_VERSION_MAJOR" -lt "\$MIN_DOCKER_VERSION_MAJOR" ] || -[ "\$DOCKER_VERSION_MAJOR" -eq "\$MIN_DOCKER_VERSION_MAJOR" -a "\$DOCKER_VERSION_MINOR" -lt "\$MIN_DOCKER_VERSION_MINOR" ]; then - echo "IPFIX_RITA requires Docker version \$MIN_DOCKER_VERSION_MAJOR.\$MIN_DOCKER_VERSION_MINOR+. Please upgrade to the latest" - echo "version of Docker using the official instructions for your OS." - exit 1 -fi - -# Ensure docker-compose is functional -if [ ! -x "\$(command -v docker-compose)" ]; then - echo "'docker-compose' was not found in the PATH. Please install the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi - -DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" -DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" - -MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 -MIN_DOCKER_COMPOSE_VERSION_MINOR=17 - -if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || -[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then - echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi - -# Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" -INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" -INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" - -echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load - -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" - -SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" -else - # TODO: set up migration - echo "Existing configuration found. Skipping..." - SETUP_CONFIG="false" -fi - -echo "Installing ipfix-rita in \$INSTALLATION_DIR" - -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" -fi - -mkdir -p "\$INSTALLATION_DIR" - -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" - -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" - -# set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf -fi - -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start - -if [ "\$SETUP_CONFIG" = "true" ]; then - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" - - echo "" - echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." - echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." - echo "In order to support this type of installation, you will need to" - echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." - echo "If needed, please do so, and restart MongoDB before continuing." - echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" - echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" - echo "better if it is installed on a machine separate from RITA/ MongoDB." - echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r - if [ -n "\$REPLY" ]; then - RITA_MONGO_URI="\$REPLY" - fi - - RITA_MONGO_AUTH="null" - - echo "" - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - done - - if [ "\$REPLY" = "2" ]; then - RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then - RITA_MONGO_AUTH="MONGODB-CR" - fi - - RITA_MONGO_TLS="false" - RITA_MONGO_TLS_CHECK_CERT="false" - RITA_MONGO_TLS_CERT_PATH="null" - echo "" - read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS="true" - RITA_MONGO_TLS_CHECK_CERT="true" - RITA_MONGO_TLS_CERT_PATH="null" - read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" - fi - - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then - read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS_CHECK_CERT="false" - fi - fi - fi - - echo "" - echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" - echo "where DBROOT consists of alphanumerics, underscores, and hyphens." - RITA_DATASET_DBROOT="IPFIX" - read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" - fi - - #unindent to ensure nothing breaks with awk - -awk -v db_root="\$RITA_DATASET_DBROOT" \\ --v mongo_uri="\$RITA_MONGO_URI" \\ --v mongo_auth="\$RITA_MONGO_AUTH" \\ --v mongo_tls_enable="\$RITA_MONGO_TLS" \\ --v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ --v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' -# flag is used to determine if we are in the right scope - -# Unset the flag if we see "abc:" or " abc:" on a line -# by itself if there are 2 or less preceding spaces -/^( )?[^ ]+:\$/{ - flag="" -} - -# Trigger the flag as we are entering the scope -/ RITA-MongoDB:/{ - flag=1 -} - -flag && NF && /ConnectionString:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_uri; - print val \$0; - next -} - -flag && NF && /AuthenticationMechanism:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_auth; - print val \$0; - next -} - -flag && NF && /Enable:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_enable; - print val \$0; - next -} - -flag && NF && /VerifyCertificate:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_cert_check; - print val \$0; - next -} - -flag && NF && /CAFile:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=mongo_tls_ca_path; - print val \$0; - next -} - -flag && NF && /DBRoot:/{ - match(\$0,/^ +/); - val=substr(\$0,RSTART,RLENGTH); - \$NF=db_root; - print val \$0; - next -} - -1 -' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ -mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml - -echo "" -echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" -echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" -echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" -echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." -fi - -echo "" -echo "IPFIX-RITA will run at start up unless the system has been stopped." -echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." -echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." -echo "To view the system logs, run 'ipfix-rita logs -f'." -echo "" - -echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." - -ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita - -echo "" -echo "Starting IPFIX-RITA..." - -"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d - -echo "The IPFIX-RITA installer has finished." - -# Change back to the old directory at the end -cd \$_OLD_DIR; unset _OLD_DIR -EOF - -chmod +x "$INSTALLER_INSTALL_SCRIPT" - tar -C $TMP_DIR -czf $INSTALLER_TARBALL $IPFIX_RITA_ARCHIVE ################################################################################ diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh new file mode 100755 index 0000000..f4ce9b5 --- /dev/null +++ b/install-scripts/helpers/check_docker-compose.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Exit values +# 0 correct version installed +# 3 not installed +# 4 older than required minimum version +# 5 newer than required maximum version + +if [ ! -x "$(command -v docker-compose)" ]; then + exit 3 +fi + +MIN_VERSION_MAJOR=1 +MIN_VERSION_MINOR=16 +MAX_VERSION_MAJOR=1 +MAX_VERSION_MINOR=22 + +VERSION="$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" +VERSION_MINOR="$(echo $VERSION | cut -d' ' -f2)" + +if [ "$VERSION_MAJOR" -lt "$MIN_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MIN_VERSION_MAJOR" -a "$VERSION_MINOR" -lt "$MIN_VERSION_MINOR" ]; then + exit 4 +elif [ "$VERSION_MAJOR" -gt "$MAX_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MAX_VERSION_MAJOR" -a "$VERSION_MINOR" -gt "$MAX_VERSION_MINOR" ]; then + exit 5 +else + exit 0 +fi diff --git a/install-scripts/helpers/check_docker.sh b/install-scripts/helpers/check_docker.sh new file mode 100755 index 0000000..a2c3465 --- /dev/null +++ b/install-scripts/helpers/check_docker.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Exit values +# 0 correct version installed +# 3 not installed +# 4 older than required minimum version +# 5 newer than required maximum version + +if [ ! -x "$(command -v docker)" ]; then + exit 3 +fi + +MIN_VERSION_MAJOR=17 +MIN_VERSION_MINOR=05 +MAX_VERSION_MAJOR=18 +MAX_VERSION_MINOR=09 + +VERSION="$(docker -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" +VERSION_MINOR="$(echo $VERSION | cut -d' ' -f2)" + +if [ "$VERSION_MAJOR" -lt "$MIN_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MIN_VERSION_MAJOR" -a "$VERSION_MINOR" -lt "$MIN_VERSION_MINOR" ]; then + exit 4 +elif [ "$VERSION_MAJOR" -gt "$MAX_VERSION_MAJOR" ] || + [ "$VERSION_MAJOR" -eq "$MAX_VERSION_MAJOR" -a "$VERSION_MINOR" -gt "$MAX_VERSION_MINOR" ]; then + exit 5 +else + exit 0 +fi diff --git a/install-scripts/install_docker.sh b/install-scripts/install_docker.sh new file mode 100755 index 0000000..d33cdbc --- /dev/null +++ b/install-scripts/install_docker.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Change dir to script dir +pushd "$(dirname "$BASH_SOURCE[0]")" > /dev/null + +# Options and Usage +# ----------------------------------- +usage() { + scriptName=$(basename "$0") + echo -n "${scriptName} [OPTION]... +Install needed docker code to support IPFIX-RITA. +Options: + -g, --group-add Add the current user to the 'docker' group + -r, --replace-shell (Implies -g) When finished, replaces the current shell + so the current user can control docker immediately. + This will prevent any calling scripts from executing further. + -h, --help Display this help and exit +" +} + +askYN () { + TESTYN="" + while [ "$TESTYN" != 'Y' ] && [ "$TESTYN" != 'N' ] ; do + echo -n '?' >&2 + read TESTYN || : + case $TESTYN in + T*|t*|Y*|y*) TESTYN='Y' ;; + F*|f*|N*|n*) TESTYN='N' ;; + esac + done + + if [ "$TESTYN" = 'Y' ]; then + return 0 #True + else + return 1 #False + fi +} #End of askYN + +ADD_DOCKER_GROUP=false + +# Parse through command args to override values +while [[ $# -gt 0 ]]; do + case $1 in + -g|--group-add) + ADD_DOCKER_GROUP=true + ;; + -r|--replace-shell) + ADD_DOCKER_GROUP=true + REPLACE_SHELL=true + ;; + -h|--help) + usage >&2 + exit + ;; + *) + ;; + esac + shift +done + +helpers/check_docker.sh +# Store the exit code +DOCKER_CHECK=$? +if [ "$DOCKER_CHECK" -gt 3 ]; then + # This may overwrite a file maintained by a package. + echo "An unsupported version of Docker appears to already be installed. It will be replaced." +fi +if [ "$DOCKER_CHECK" -eq 0 ]; then + echo "Docker appears to already be installed. Skipping." +elif [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then + #This configuration file is used in both Redhat RHEL and Centos distributions, so we're running under RHEL/Centos 7.x + # https://docs.docker.com/engine/installation/linux/docker-ce/centos/ + + sudo yum makecache fast + + if rpm -q docker >/dev/null 2>&1 || rpm -q docker-common >/dev/null 2>&1 || rpm -q docker-selinux >/dev/null 2>&1 || rpm -q docker-engine >/dev/null 2>&1 ; then + echo -n "One or more of these packages are installed: docker, docker-common, docker-selinux, and/or docker-engine . The docker website encourages us to remove these before installing docker-ce. Would you like to remove these older packages (recommended: yes)" + if askYN ; then + sudo yum -y remove docker docker-common docker-selinux docker-engine + else + echo "You chose not to remove the older docker packages. The install may not succeed." + fi + fi + + sudo yum install -y yum-utils device-mapper-persistent-data lvm2 shadow-utils + + sudo yum-config-manager --enable extras + + if [ ! -f /etc/yum.repos.d/docker-ce.repo ]; then + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + fi + + sudo wget -q https://download.docker.com/linux/centos/gpg -O ~/DOCKER-GPG-KEY + sudo rpm --import ~/DOCKER-GPG-KEY + + sudo yum -y install docker-ce + + sudo systemctl start docker + sudo systemctl enable docker +elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then + ### Install Docker on Ubuntu ### + # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-repository + + echo "Installing Docker package repo..." + sudo apt-get update + sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + + echo "Installing latest Docker version..." + sudo apt-get update + sudo apt-get install -y docker-ce + sudo service docker start +else + echo "This system does not appear to be a Centos 7.x, RHEL 7.x, or Ubuntu Linux system. Unable to install docker." + exit 1 +fi + +helpers/check_docker-compose.sh +# Store the exit code +DOCKER_COMPOSE_CHECK=$? +if [ "$DOCKER_COMPOSE_CHECK" -gt 3 ]; then + # This may overwrite a file maintained by a package. + echo "An unsupported version of Docker-Compose appears to already be installed. It will be replaced." +fi +if [ "$DOCKER_COMPOSE_CHECK" -eq 0 ]; then + echo "Docker-Compose appears to already be installed. Skipping." +else + ### Install Docker-Compose ### + # https://docs.docker.com/compose/install/#install-compose + DOCKER_COMPOSE_VERSION="1.21.2" + + echo "Installing Docker-Compose v${DOCKER_COMPOSE_VERSION}..." + sudo curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o /usr/bin/docker-compose + sudo chmod +x /usr/bin/docker-compose +fi + +if [ "${ADD_DOCKER_GROUP}" = "true" ]; then + # Add current user to docker group + echo "Adding current user to docker group..." + #sudo groupadd docker + sudo usermod -aG docker $USER + + if [ "${REPLACE_SHELL}" = "true" ]; then + echo "Docker installation complete. You should have access to the 'docker' and 'docker-compose' commands immediately." + # Hack to activate the docker group on the current user without logging out. + # Downside is it completely replaces the shell and prevents calling scripts from continuing. + # https://superuser.com/a/853897 + exec sg docker newgrp `id -gn` + fi + + echo "You will need to login again for these changes to take effect." + echo "Docker installation complete. You should have access to the 'docker' and 'docker-compose' commands once you log out and back in." +else + echo "Docker installation complete. 'docker' and 'docker-compose' must be run using sudo or the root account." +fi + +# Change back to original directory +popd > /dev/null diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh new file mode 100755 index 0000000..4ffba60 --- /dev/null +++ b/install-scripts/install_ipfix-rita.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash + +askYN () { + TESTYN="" + while [ "$TESTYN" != 'Y' ] && [ "$TESTYN" != 'N' ] ; do + echo -n '?' >&2 + read TESTYN || : + case $TESTYN in + T*|t*|Y*|y*) TESTYN='Y' ;; + F*|f*|N*|n*) TESTYN='N' ;; + esac + done + + if [ "$TESTYN" = 'Y' ]; then + return 0 #True + else + return 1 #False + fi +} #End of askYN + +# Stop if there are any errors +set -e +# Change dir to script dir +_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; + +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run with administrator privileges." + exit 1 +fi + +# Install docker if needed +chmod +x "scripts/install_docker.sh" +sudo scripts/install_docker.sh +require_file /usr/bin/docker || fail "Docker does not appear to have been installed successfully" +if sudo docker run hello-world 2>/dev/null | grep -iq 'Hello from Docker!' ; then + status 'Docker appears to be working, continuing.' +else + echo 'Docker does not appear to be able to pull down a docker instance and run it successfully.' >&2 + echo 'The most likely explanation is that this system is not able to make outbound connections to the Internet.' >&2 + echo 'We recommend you fix this, but will proceed anyways in 20 seconds.' >&2 + sleep 20 +fi + +# Ensure docker-compose is functional +#TODO If we install docker automatically intall docker-compose automatically too +if [ ! -x "\$(command -v docker-compose)" ]; then + echo "'docker-compose' was not found in the PATH. Please install the latest" + echo "version of docker-compose using the official instructions for your OS." + exit 1 +fi + +DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" +DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" +DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" + +MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 +MIN_DOCKER_COMPOSE_VERSION_MINOR=17 + +if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || +[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then + echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" + echo "version of docker-compose using the official instructions for your OS." + exit 1 +fi +#end of TODO's + +# Set by make-release +INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" +INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" +INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" +DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + +echo "Loading IPFIX-RITA Docker images... This may take a few minutes." +gzip -d -c \${DOCKER_IMAGES} | docker load + +echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + +SETUP_CONFIG="true" +if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +else + # TODO: set up migration + echo "Existing configuration found. Skipping..." + SETUP_CONFIG="false" +fi + +echo "Installing ipfix-rita in \$INSTALLATION_DIR" + +if [ -d "\$INSTALLATION_DIR" ]; then + rm -rf "\$INSTALLATION_DIR" +fi + +mkdir -p "\$INSTALLATION_DIR" + +cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" +chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + +cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + +# set receive buffer size for logstash collector +RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" +if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +fi + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + +if [ "\$SETUP_CONFIG" = "true" ]; then + IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + + echo "" + echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." + echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." + echo "In order to support this type of installation, you will need to" + echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." + echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" + echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" + echo "better if it is installed on a machine separate from RITA/ MongoDB." + echo "" + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + if [ -n "\$REPLY" ]; then + RITA_MONGO_URI="\$REPLY" + fi + + RITA_MONGO_AUTH="null" + + echo "" + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + + while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + done + + if [ "\$REPLY" = "2" ]; then + RITA_MONGO_AUTH="SCRAM-SHA-1" + elif [ "\$REPLY" = "3" ]; then + RITA_MONGO_AUTH="MONGODB-CR" + fi + + RITA_MONGO_TLS="false" + RITA_MONGO_TLS_CHECK_CERT="false" + RITA_MONGO_TLS_CERT_PATH="null" + echo "" + read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS="true" + RITA_MONGO_TLS_CHECK_CERT="true" + RITA_MONGO_TLS_CERT_PATH="null" + read -p "Would you like to provide a certificate authority? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + read -p "CA Path: " + RITA_MONGO_TLS_CERT_PATH="\$REPLY" + fi + + if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + read -p "Would you like to disable certificate checks? [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS_CHECK_CERT="false" + fi + fi + fi + + echo "" + echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" + echo "where DBROOT consists of alphanumerics, underscores, and hyphens." + RITA_DATASET_DBROOT="IPFIX" + read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r + if [ -n "\$REPLY" ]; then + RITA_DATASET_DBROOT="\$REPLY" + fi + + #unindent to ensure nothing breaks with awk + +awk -v db_root="\$RITA_DATASET_DBROOT" \\ +-v mongo_uri="\$RITA_MONGO_URI" \\ +-v mongo_auth="\$RITA_MONGO_AUTH" \\ +-v mongo_tls_enable="\$RITA_MONGO_TLS" \\ +-v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ +-v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' +# flag is used to determine if we are in the right scope + +# Unset the flag if we see "abc:" or " abc:" on a line +# by itself if there are 2 or less preceding spaces +/^( )?[^ ]+:\$/{ + flag="" +} + +# Trigger the flag as we are entering the scope +/ RITA-MongoDB:/{ + flag=1 +} + +flag && NF && /ConnectionString:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_uri; + print val \$0; + next +} + +flag && NF && /AuthenticationMechanism:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_auth; + print val \$0; + next +} + +flag && NF && /Enable:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_enable; + print val \$0; + next +} + +flag && NF && /VerifyCertificate:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_cert_check; + print val \$0; + next +} + +flag && NF && /CAFile:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=mongo_tls_ca_path; + print val \$0; + next +} + +flag && NF && /DBRoot:/{ + match(\$0,/^ +/); + val=substr(\$0,RSTART,RLENGTH); + \$NF=db_root; + print val \$0; + next +} + +1 +' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml + +echo "" +echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" +echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" +echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" +echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." +fi + +echo "" +echo "IPFIX-RITA will run at start up unless the system has been stopped." +echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." +echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." +echo "To view the system logs, run 'ipfix-rita logs -f'." +echo "" + +echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." + +ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita + +echo "" +echo "Starting IPFIX-RITA..." + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d + +echo "The IPFIX-RITA installer has finished." + +# Change back to the old directory at the end +cd \$_OLD_DIR; unset _OLD_DIR From dbfc51f430e685156d81d1ba00d6a75c3056532e Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Sat, 12 Jan 2019 11:10:28 -0700 Subject: [PATCH 10/19] Moving code to different sections to enable a non-interactive version of the installer. Still in the testing/debugging stages --- .../helpers/check_docker-compose.sh | 2 +- install-scripts/install_ipfix-rita.sh | 489 ++++++++++++------ 2 files changed, 340 insertions(+), 151 deletions(-) diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh index f4ce9b5..b7570fe 100755 --- a/install-scripts/helpers/check_docker-compose.sh +++ b/install-scripts/helpers/check_docker-compose.sh @@ -11,7 +11,7 @@ if [ ! -x "$(command -v docker-compose)" ]; then fi MIN_VERSION_MAJOR=1 -MIN_VERSION_MINOR=16 +MIN_VERSION_MINOR=17 MAX_VERSION_MAJOR=1 MAX_VERSION_MINOR=22 diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 4ffba60..ca59759 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -18,169 +18,262 @@ askYN () { fi } #End of askYN -# Stop if there are any errors -set -e -# Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; +check_remote_mongo() { + hostip=$1 + if [`echo 'show dbs' | mongo --host $hostip | grep '^bye'` = 'bye']; then + return 0 #True + else + return 1 #False + fi +} #End of check_remote_mongo + +can_ssh () { + #Test that we can reach the target system over ssh. + success_code=1 + if [ "$1" = "127.0.0.1" ]; then + success_code=0 + elif [ -n "$1" ]; then + token="$RANDOM.$RANDOM" + echo "Attempting to verify that we can ssh to $@ - you may need to provide a password to access this system." >&2 + ssh_out=`ssh "$@" '/bin/echo '"$token"` + if [ "$token" = "$ssh_out" ]; then + #echo "successfully connected to $@" + success_code=0 + #else + #echo "cannot connect to $@" + fi + else + echo "Please supply an ssh target as a command line parameter to can_ssh" + fi -if [[ \$EUID -ne 0 ]]; then - echo "This script must be run with administrator privileges." - exit 1 -fi + return $success_code +} #End of can_ssh -# Install docker if needed -chmod +x "scripts/install_docker.sh" -sudo scripts/install_docker.sh -require_file /usr/bin/docker || fail "Docker does not appear to have been installed successfully" -if sudo docker run hello-world 2>/dev/null | grep -iq 'Hello from Docker!' ; then - status 'Docker appears to be working, continuing.' -else - echo 'Docker does not appear to be able to pull down a docker instance and run it successfully.' >&2 - echo 'The most likely explanation is that this system is not able to make outbound connections to the Internet.' >&2 - echo 'We recommend you fix this, but will proceed anyways in 20 seconds.' >&2 - sleep 20 -fi +do_system_tests () { + while [ -n "$1" ]; do + if [ "$1" = "check_sse4_2" ]; then + status "Hardware tests" + require_file /proc/cpuinfo || fail "Missing /proc/cpuinfo - is this a Linux system?" + fi + shift + done -# Ensure docker-compose is functional -#TODO If we install docker automatically intall docker-compose automatically too -if [ ! -x "\$(command -v docker-compose)" ]; then - echo "'docker-compose' was not found in the PATH. Please install the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi + for one_dir in $HOME / /usr ; do + #"tr" in next command removes the linefeed at the end of the number of free megabytes (and any other whitespace) + [ `df -h $one_dir --output=avail --block-size=1048576 | grep -v '^Avail' | tr -dc '[0-9]'` -gt 5000 ] || fail "${one_dir} has less than 5GB free space" + done -DOCKER_COMPOSE_VERSION="\$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" -DOCKER_COMPOSE_VERSION_MAJOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f1)" -DOCKER_COMPOSE_VERSION_MINOR="\$(echo \$DOCKER_COMPOSE_VERSION | cut -d' ' -f2)" + status "Installed software tests" + if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then + #We have apt-get , good. + sudo apt-get -qq update > /dev/null 2>&1 + inst_retcode=100000 + while [ "$inst_retcode" -gt 0 ]; do + sudo apt-get -y install gdb git wget curl make realpath lsb-release rsync tar + inst_retcode=$? + if [ "$inst_retcode" -gt 0 ]; then + echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" + sleep 60 + fi + done + #TODO test IPFIX-RITA on Contos/Red Hat System + #elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then + # if [ ! -x /bin/yum-config-manager ]; then + # sudo yum -y install yum-utils + # fi + #We have yum, good. + # sudo yum -q makecache > /dev/null 2>&1 + # #FIXME - put in place a similar loop like above for apt-get + # sudo yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar + else + fail "(apt-get and dpkg-query) is installed on this system"#nor (yum, rpm, and yum-config-manager) is installed on the system" + fi + require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" -MIN_DOCKER_COMPOSE_VERSION_MAJOR=1 -MIN_DOCKER_COMPOSE_VERSION_MINOR=17 + if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then + #echo "Centos or Redhat 7 installation detected, good." + fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then + echo "Ubuntu Linux installation detected, good." + else + fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + fi -if [ "\$DOCKER_COMPOSE_VERSION_MAJOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" ] || -[ "\$DOCKER_COMPOSE_VERSION_MAJOR" -eq "\$MIN_DOCKER_COMPOSE_VERSION_MAJOR" -a "\$DOCKER_COMPOSE_VERSION_MINOR" -lt "\$MIN_DOCKER_COMPOSE_VERSION_MINOR" ]; then - echo "IPFIX-RITA requires docker-compose version \$MIN_DOCKER_COMPOSE_VERSION_MAJOR.\$MIN_DOCKER_COMPOSE_VERSION_MINOR+. Please upgrade to the latest" - echo "version of docker-compose using the official instructions for your OS." - exit 1 -fi -#end of TODO's + return 0 +} #End of do_system_tests + +validated_ssh_target () { + #Return an ssh target that we've confirmed we can reach. "$2" is the initial target to try, and may be replaced. + #The potential target - both as supplied as parameter 2 and as returned at the end - may be blank, indicating "do not install this" + target_type="$1" + potential_target="$2" + + echo 'About to execute commands on '"$potential_target"' . You may be prompted one or more times for the password for this system.' >&2 + echo 'Note: if you are using an ssh client other than openssh and have not set up ssh key access to this system, you may be prompted for your password multiple times.' >&2 + while [ -n "$potential_target" ] && ! can_ssh "$potential_target" ; do + echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to install on this system. Just press enter if you do not wish to set up IPFIX logging. Press Ctrl-C to exit." >&2 + read potential_target <&2 + if [ -n "$potential_target" -a "$potential_target" != '127.0.0.1' ]; then + echo "Do you access that system with a username different than ${USER}? If so, enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read potential_user <&2 + if [ -n "$potential_user" ]; then + potential_target="${potential_user}@${potential_target}" + fi + fi + done -# Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" -INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" -INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + echo "$potential_target" +} #End of validated_ssh_tartget + +get_line() { + rita_file=$1 + search_key=$2 + + #get the line that matches the search key that isn't commented out + line=`cat $rita_file | grep $search_key | grep -v "#"` + +} #End of get line + +get_rita_data_interactive() { + IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + + echo "" + echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." + echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." + echo "In order to support this type of installation, you will need to" + echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." + echo "If needed, please do so, and restart MongoDB before continuing." + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" + echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" + echo "better if it is installed on a machine separate from RITA/ MongoDB." + echo "" + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + read RITA_MONGO_URI <&2 + if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then + echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 + if askYN ; then + echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read rita_user <&2 + if [ -n "$rita_user" ]; then + rita_system="${rita_user}@${rita_system}" + fi + fi + rita_system=`validated_ssh_target rita "$rita_system"` + else + rita_system="127.0.0.1" + fi -echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load + RITA_MONGO_AUTH="null" + #instead of prompting get the RITA address and split from there to fill in our + # config file + echo "" + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + + while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + echo "Which authentication scheme should be used to contact the database if any? [None]" + echo "1) None" + echo "2) SCRAM-SHA-1" + echo "3) MONGODB-CR" + done -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + if [ "\$REPLY" = "2" ]; then + RITA_MONGO_AUTH="SCRAM-SHA-1" + elif [ "\$REPLY" = "3" ]; then + RITA_MONGO_AUTH="MONGODB-CR" + fi -SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" -else - # TODO: set up migration - echo "Existing configuration found. Skipping..." - SETUP_CONFIG="false" -fi + RITA_MONGO_TLS="false" + RITA_MONGO_TLS_CHECK_CERT="false" + RITA_MONGO_TLS_CERT_PATH="null" + echo "" + read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS="true" + RITA_MONGO_TLS_CHECK_CERT="true" + RITA_MONGO_TLS_CERT_PATH="null" + read -p "Would you like to provide a certificate authority? (y/n) [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + read -p "CA Path: " + RITA_MONGO_TLS_CERT_PATH="\$REPLY" + fi + + if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + read -p "Would you like to disable certificate checks? [n] " -r + if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + RITA_MONGO_TLS_CHECK_CERT="false" + fi + fi + fi -echo "Installing ipfix-rita in \$INSTALLATION_DIR" + echo "" + echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" + echo "where DBROOT consists of alphanumerics, underscores, and hyphens." + RITA_DATASET_DBROOT="IPFIX" + read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r + if [ -n "\$REPLY" ]; then + RITA_DATASET_DBROOT="\$REPLY" + fi -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" -fi + write_converter_conf $RITA_DATASET_DBROOT $RITA_MONGO_URI $RITA_MONGO_AUTH $RITA_MONGO_TLS $RITA_MONGO_TLS_CHECK_CERT $RITA_MONGO_TLS_CERT_PATH +} #End of get_rita_data_interactive + +get_rita_data_noninteractive() { + rita_system=$1 + if [ -n "$rita_system" -a "$rita_system" != '127.0.0.1' ]; then + echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 + if askYN ; then + echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 + read rita_user <&2 + if [ -n "$rita_user" ]; then + rita_system="${rita_user}@${rita_system}" + fi + fi + rita_system=`validated_ssh_target rita "$rita_system"` + else + rita_system="127.0.0.1" + fi -mkdir -p "\$INSTALLATION_DIR" + if [ "$rita_system" != '127.0.0.1' ]; then + scp $rita_system:/etc/rita/config.yaml . + rita_conf="./config.yaml" + delete_conf=true + else + #do something great here + rita_conf="/etc/rita/config.yaml" + fi -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + db_root=${rita_system#*@} + mongo_uri="mongodb://$db_root:$MONGO_PORT" + mongo_auth=`get_line $rita_conf "AuthenticationMechanism: "` + mongo_tls_enable=`get_line $rita_conf "Enable: "` + mongo_tls_cert_check=`get_line $rita_conf "VerifyCertificate: "` + mongo_tls_ca_path=`get_line $rita_conf "CAFile: "` -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + #If the config file was copied from a remote server, delete that file + if [ "$delete_conf" != "true" ]; then + rm "./config.yaml" + fi -# set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf -fi + #If the config specified a CA file, let's copy that + if [[ $mongo_tls_ca_path != *"null" ]]; then + ca_file=${mongo_tls_ca_path#"CAFile: "} + scp $rita_system:$ca_file $ca_file + fi -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + write_converter_conf $db_root $mongo_uri $mongo_auth $mongo_tls_enable $mongo_tls_cert_check $mongo_tls_ca_path +} #End of get_rita_data_noninteractive -if [ "\$SETUP_CONFIG" = "true" ]; then - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" - - echo "" - echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." - echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." - echo "In order to support this type of installation, you will need to" - echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." - echo "If needed, please do so, and restart MongoDB before continuing." - echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" - echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" - echo "better if it is installed on a machine separate from RITA/ MongoDB." - echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r - if [ -n "\$REPLY" ]; then - RITA_MONGO_URI="\$REPLY" - fi - - RITA_MONGO_AUTH="null" - - echo "" - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - done - - if [ "\$REPLY" = "2" ]; then - RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then - RITA_MONGO_AUTH="MONGODB-CR" - fi - - RITA_MONGO_TLS="false" - RITA_MONGO_TLS_CHECK_CERT="false" - RITA_MONGO_TLS_CERT_PATH="null" - echo "" - read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS="true" - RITA_MONGO_TLS_CHECK_CERT="true" - RITA_MONGO_TLS_CERT_PATH="null" - read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" - fi - - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then - read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then - RITA_MONGO_TLS_CHECK_CERT="false" - fi - fi - fi - - echo "" - echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" - echo "where DBROOT consists of alphanumerics, underscores, and hyphens." - RITA_DATASET_DBROOT="IPFIX" - read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" - fi - - #unindent to ensure nothing breaks with awk +write_converter_conf() { + RITA_DATASET_DBROOT=$1 + RITA_MONGO_URI=$2 + RITA_MONGO_AUTH=$3 + RITA_MONGO_TLS=$4 + RITA_MONGO_TLS_CHECK_CERT=$5 + RITA_MONGO_TLS_CERT_PATH=$6 awk -v db_root="\$RITA_DATASET_DBROOT" \\ -v mongo_uri="\$RITA_MONGO_URI" \\ @@ -252,12 +345,108 @@ flag && NF && /DBRoot:/{ 1 ' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml +} #End of write_converter_conf -echo "" -echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" -echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" -echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" -echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." +# Stop if there are any errors +set -e +# Change dir to script dir +_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; + +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run with administrator privileges." + exit 1 +fi + +# Parse through command args to override values +echo "Loading command line params" >&2 +if [ "$1" = 'help' -o "$1" = '--help' ]; then + usage_text + exit 1 +fi + +while [ -n "$1" ]; do + case "$1" in + rita|Rita|RITA) + if [ -n "$2" ] && ! echo "$2" | egrep -iq '(^rita$|^aih$)' ; then + rita_system="$2" + shift + else + rita_system='127.0.0.1' + fi + ;; + *) + echo "Unrecognized parameter $1" >&2 + usage_text + exit 1 + ;; + esac + shift +done + +#TODO check system requirements + +# Install docker if needed +chmod +x "scripts/install_docker.sh" +sudo scripts/install_docker.sh + +# Set by make-release +INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" +INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" +INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" +DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" + +echo "Loading IPFIX-RITA Docker images... This may take a few minutes." +gzip -d -c \${DOCKER_IMAGES} | docker load + +echo "Installing configuration files to \$INSTALLATION_ETC_DIR" + +SETUP_CONFIG="true" +if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +else + # TODO: set up migration + echo "Existing configuration found. Skipping..." + SETUP_CONFIG="false" +fi + +echo "Installing ipfix-rita in \$INSTALLATION_DIR" + +if [ -d "\$INSTALLATION_DIR" ]; then + rm -rf "\$INSTALLATION_DIR" +fi + +mkdir -p "\$INSTALLATION_DIR" + +cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" +chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" + +cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" + +# set receive buffer size for logstash collector +RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" +if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +fi + +"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start + +if [ "\$SETUP_CONFIG" = "true" ]; then + #From here out we want to move to a interactive/active install command + if [ -n "$rita_system" ]; then + get_rita_data_noninteractive $rita_system + else + get_rita_data_interactive + fi + +#We have written to config + echo "" + echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" + echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" + echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" + echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." fi echo "" From a94e7dcf03ea7c352d89b65f6623a7f8360c2179 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 15 Jan 2019 16:34:03 -0700 Subject: [PATCH 11/19] Made some modifications to the install script, still needs debugging and additional work --- install-scripts/install_ipfix-rita.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index ca59759..045940b 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -27,6 +27,8 @@ check_remote_mongo() { fi } #End of check_remote_mongo + + can_ssh () { #Test that we can reach the target system over ssh. success_code=1 @@ -49,6 +51,21 @@ can_ssh () { return $success_code } #End of can_ssh + + +fail () { + echo "$* ." >&2 + echo "This is normally an unrecoverable problem, and we recommend fixing the problem and restarting the script. Please contact technical support for help in resolving the issue. If you feel the script should continue, enter Y and the script will attempt to finish. Entering N will cause this script to exit." >&2 + if askYN ; then + echo "Script will continue at user request. This may not result in a working configuration." >&2 + sleep 5 + else + exit 1 + fi +} #end of fail + + + do_system_tests () { while [ -n "$1" ]; do if [ "$1" = "check_sse4_2" ]; then @@ -91,8 +108,7 @@ do_system_tests () { require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then - #echo "Centos or Redhat 7 installation detected, good." - fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" + echo "Centos or Redhat 7 installation detected, good." elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then echo "Ubuntu Linux installation detected, good." else From 6e56f069cfcf6e78ed67d20f56abda754435fd7b Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Wed, 16 Jan 2019 01:01:31 -0700 Subject: [PATCH 12/19] Adding changes to make the install interactive functional. Need to add mongo check and test/debug non-interactive commands --- dev-scripts/make-release.sh | 3 + .../helpers/check_docker-compose.sh | 2 +- install-scripts/install_ipfix-rita.sh | 110 +++++++++--------- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index e017dcc..09efde7 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -64,6 +64,9 @@ fi # Insert the install scripts # Start by adding the base install command cp "$INSTALLER_SCRIPTS_DIR/install_ipfix-rita.sh" "$INSTALLER_DIR" +sed -i "s|INSTALLATION_DIR=\"REPLACE_WITH_INSTALL_DIR\"|INSTALLATION_DIR=\"$INSTALLATION_DIR\"|g" $INSTALLER_DIR/install_ipfix-rita.sh +sed -i "s|INSTALLATION_ETC_DIR=\"REPLACE_WITH_ETC_DIR\"|INSTALLATION_ETC_DIR=\"$INSTALLATION_ETC_DIR\"|g" $INSTALLER_DIR/install_ipfix-rita.sh +sed -i "s|DOCKER_IMAGES=\"./REPLACE_WITH_TARBALL\"|DOCKER_IMAGES=\"$DOCKER_IMAGE_OUT\"|g" $INSTALLER_DIR/install_ipfix-rita.sh #Then add all the helper scirpts to the tarball and remove install-ipfix-rita.sh cp -r "$INSTALLER_SCRIPTS_DIR" "$INSTALLER_DIR" rm "$INSTALLER_DIR/install-scripts/install_ipfix-rita.sh" diff --git a/install-scripts/helpers/check_docker-compose.sh b/install-scripts/helpers/check_docker-compose.sh index b7570fe..e7cc6ae 100755 --- a/install-scripts/helpers/check_docker-compose.sh +++ b/install-scripts/helpers/check_docker-compose.sh @@ -13,7 +13,7 @@ fi MIN_VERSION_MAJOR=1 MIN_VERSION_MINOR=17 MAX_VERSION_MAJOR=1 -MAX_VERSION_MINOR=22 +MAX_VERSION_MINOR=23 VERSION="$(docker-compose -v | sed 's/^.* \([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/')" VERSION_MAJOR="$(echo $VERSION | cut -d' ' -f1)" diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 045940b..21632c3 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -103,7 +103,7 @@ do_system_tests () { # #FIXME - put in place a similar loop like above for apt-get # sudo yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar else - fail "(apt-get and dpkg-query) is installed on this system"#nor (yum, rpm, and yum-config-manager) is installed on the system" + fail "(apt-get and dpkg-query) is installed on this system" fi require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" @@ -127,7 +127,7 @@ validated_ssh_target () { echo 'About to execute commands on '"$potential_target"' . You may be prompted one or more times for the password for this system.' >&2 echo 'Note: if you are using an ssh client other than openssh and have not set up ssh key access to this system, you may be prompted for your password multiple times.' >&2 while [ -n "$potential_target" ] && ! can_ssh "$potential_target" ; do - echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to install on this system. Just press enter if you do not wish to set up IPFIX logging. Press Ctrl-C to exit." >&2 + echo "What is the hostname or IP address where you would like to send IPFIX logs to? Enter 127.0.0.1 if you want to run on this system. Press Ctrl-C to exit." >&2 read potential_target <&2 if [ -n "$potential_target" -a "$potential_target" != '127.0.0.1' ]; then echo "Do you access that system with a username different than ${USER}? If so, enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 @@ -151,8 +151,8 @@ get_line() { } #End of get line get_rita_data_interactive() { - IPFIX_RITA_NETWORK_GATEWAY=\$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") - RITA_MONGO_URI="mongodb://\$IPFIX_RITA_NETWORK_GATEWAY:27017" + IPFIX_RITA_NETWORK_GATEWAY=$(docker inspect ipfix_rita_default --format "{{with (index .IPAM.Config 0)}}{{.Gateway}}{{end}}") + RITA_MONGO_URI="mongodb://$IPFIX_RITA_NETWORK_GATEWAY:27017" echo "" echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." @@ -164,7 +164,7 @@ get_rita_data_interactive() { echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" echo "better if it is installed on a machine separate from RITA/ MongoDB." echo "" - read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [\$RITA_MONGO_URI]: " -r + read -p "What MongoDB URI should IPFIX-RITA use to contact the RITA database [$RITA_MONGO_URI]: " -r read RITA_MONGO_URI <&2 if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 @@ -180,6 +180,8 @@ get_rita_data_interactive() { rita_system="127.0.0.1" fi + #TODO check if the mongo server is available + RITA_MONGO_AUTH="null" #instead of prompting get the RITA address and split from there to fill in our # config file @@ -189,16 +191,16 @@ get_rita_data_interactive() { echo "2) SCRAM-SHA-1" echo "3) MONGODB-CR" - while read && [[ ! ( "\$REPLY" =~ ^[123]\$ || -z "\$REPLY" ) ]]; do + while read && [[ ! ( "$REPLY" =~ ^[123]$ || -z "$REPLY" ) ]]; do echo "Which authentication scheme should be used to contact the database if any? [None]" echo "1) None" echo "2) SCRAM-SHA-1" echo "3) MONGODB-CR" done - if [ "\$REPLY" = "2" ]; then + if [ "$REPLY" = "2" ]; then RITA_MONGO_AUTH="SCRAM-SHA-1" - elif [ "\$REPLY" = "3" ]; then + elif [ "$REPLY" = "3" ]; then RITA_MONGO_AUTH="MONGODB-CR" fi @@ -207,19 +209,19 @@ get_rita_data_interactive() { RITA_MONGO_TLS_CERT_PATH="null" echo "" read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS="true" RITA_MONGO_TLS_CHECK_CERT="true" RITA_MONGO_TLS_CERT_PATH="null" read -p "Would you like to provide a certificate authority? (y/n) [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then read -p "CA Path: " - RITA_MONGO_TLS_CERT_PATH="\$REPLY" + RITA_MONGO_TLS_CERT_PATH="$REPLY" fi - if [ "\$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then + if [ "$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then read -p "Would you like to disable certificate checks? [n] " -r - if [[ "\$REPLY" =~ ^[Yy]\$ ]]; then + if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS_CHECK_CERT="false" fi fi @@ -230,8 +232,8 @@ get_rita_data_interactive() { echo "where DBROOT consists of alphanumerics, underscores, and hyphens." RITA_DATASET_DBROOT="IPFIX" read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r - if [ -n "\$REPLY" ]; then - RITA_DATASET_DBROOT="\$REPLY" + if [ -n "$REPLY" ]; then + RITA_DATASET_DBROOT="$REPLY" fi write_converter_conf $RITA_DATASET_DBROOT $RITA_MONGO_URI $RITA_MONGO_AUTH $RITA_MONGO_TLS $RITA_MONGO_TLS_CHECK_CERT $RITA_MONGO_TLS_CERT_PATH @@ -291,12 +293,12 @@ write_converter_conf() { RITA_MONGO_TLS_CHECK_CERT=$5 RITA_MONGO_TLS_CERT_PATH=$6 -awk -v db_root="\$RITA_DATASET_DBROOT" \\ --v mongo_uri="\$RITA_MONGO_URI" \\ --v mongo_auth="\$RITA_MONGO_AUTH" \\ --v mongo_tls_enable="\$RITA_MONGO_TLS" \\ --v mongo_tls_cert_check="\$RITA_MONGO_TLS_CHECK_CERT" \\ --v mongo_tls_ca_path="\$RITA_MONGO_TLS_CERT_PATH" ' +awk -v db_root="$RITA_DATASET_DBROOT" \ +-v mongo_uri="$RITA_MONGO_URI" \ +-v mongo_auth="$RITA_MONGO_AUTH" \ +-v mongo_tls_enable="$RITA_MONGO_TLS" \ +-v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ +-v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' # flag is used to determine if we are in the right scope # Unset the flag if we see "abc:" or " abc:" on a line @@ -359,16 +361,16 @@ flag && NF && /DBRoot:/{ } 1 -' \$INSTALLATION_ETC_DIR/converter/converter.yaml > \$INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ -mv \$INSTALLATION_ETC_DIR/converter/converter-new.yaml \$INSTALLATION_ETC_DIR/converter/converter.yaml +' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors set -e # Change dir to script dir -_OLD_DIR=\$(pwd); cd "\$(dirname "\$BASH_SOURCE[0]")"; +_OLD_DIR=$(pwd); cd "$(dirname "$BASH_SOURCE[0]")"; -if [[ \$EUID -ne 0 ]]; then +if [[ $EUID -ne 0 ]]; then echo "This script must be run with administrator privileges." exit 1 fi @@ -402,54 +404,54 @@ done #TODO check system requirements # Install docker if needed -chmod +x "scripts/install_docker.sh" -sudo scripts/install_docker.sh +chmod +x "install-scripts/install_docker.sh" +sudo install-scripts/install_docker.sh # Set by make-release -INSTALLATION_DIR="$INSTALLATION_DIR" +INSTALLATION_DIR="REPLACE_WITH_INSTALL_DIR" INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="$INSTALLATION_ETC_DIR" -DOCKER_IMAGES="./$DOCKER_IMAGE_OUT" +INSTALLATION_ETC_DIR="REPLACE_WITH_ETC_DIR" +DOCKER_IMAGES="./REPLACE_WITH_TARBALL" echo "Loading IPFIX-RITA Docker images... This may take a few minutes." -gzip -d -c \${DOCKER_IMAGES} | docker load +gzip -d -c ${DOCKER_IMAGES} | docker load -echo "Installing configuration files to \$INSTALLATION_ETC_DIR" +echo "Installing configuration files to $INSTALLATION_ETC_DIR" SETUP_CONFIG="true" -if [ ! -d "\$INSTALLATION_ETC_DIR" ]; then - cp -r pkg/etc "\$INSTALLATION_ETC_DIR" +if [ ! -d "$INSTALLATION_ETC_DIR" ]; then + cp -r pkg/etc "$INSTALLATION_ETC_DIR" else # TODO: set up migration echo "Existing configuration found. Skipping..." SETUP_CONFIG="false" fi -echo "Installing ipfix-rita in \$INSTALLATION_DIR" +echo "Installing ipfix-rita in $INSTALLATION_DIR" -if [ -d "\$INSTALLATION_DIR" ]; then - rm -rf "\$INSTALLATION_DIR" +if [ -d "$INSTALLATION_DIR" ]; then + rm -rf "$INSTALLATION_DIR" fi -mkdir -p "\$INSTALLATION_DIR" +mkdir -p "$INSTALLATION_DIR" -cp -r ./pkg/bin "\$INSTALLATION_BIN_DIR" -chmod +x "\$INSTALLATION_BIN_DIR/ipfix-rita" +cp -r ./pkg/bin "$INSTALLATION_BIN_DIR" +chmod +x "$INSTALLATION_BIN_DIR/ipfix-rita" -cp -r ./pkg/lib "\$INSTALLATION_LIB_DIR" +cp -r ./pkg/lib "$INSTALLATION_LIB_DIR" # set receive buffer size for logstash collector -RECV_BUFF_SIZE=\$(sysctl -n net.core.rmem_max) -RECV_BUFF_OPT_SIZE="\$((1024*1024*64))" -if [ "\$RECV_BUFF_SIZE" -lt "\$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=\$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=\$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf +RECV_BUFF_SIZE=$(sysctl -n net.core.rmem_max) +RECV_BUFF_OPT_SIZE="$((1024*1024*64))" +if [ "$RECV_BUFF_SIZE" -lt "$RECV_BUFF_OPT_SIZE" ]; then + sysctl -w net.core.rmem_max=$RECV_BUFF_OPT_SIZE + echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf fi -"\$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start +"$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start -if [ "\$SETUP_CONFIG" = "true" ]; then +if [ "$SETUP_CONFIG" = "true" ]; then #From here out we want to move to a interactive/active install command if [ -n "$rita_system" ]; then get_rita_data_noninteractive $rita_system @@ -459,10 +461,10 @@ if [ "\$SETUP_CONFIG" = "true" ]; then #We have written to config echo "" - echo "Your settings have been saved to \$INSTALLATION_ETC_DIR/converter/converter.yaml" + echo "Your settings have been saved to $INSTALLATION_ETC_DIR/converter/converter.yaml" echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" - echo "in \$INSTALLATION_ETC_DIR/converter/converter.yaml." + echo "in $INSTALLATION_ETC_DIR/converter/converter.yaml." fi echo "" @@ -472,16 +474,16 @@ echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." echo "To view the system logs, run 'ipfix-rita logs -f'." echo "" -echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to \$INSTALLATION_BIN_DIR/ipfix-rita." +echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to $INSTALLATION_BIN_DIR/ipfix-rita." -ln -fs "\$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita +ln -fs "$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita echo "" echo "Starting IPFIX-RITA..." -"\$INSTALLATION_BIN_DIR/ipfix-rita" up -d +"$INSTALLATION_BIN_DIR/ipfix-rita" up -d echo "The IPFIX-RITA installer has finished." # Change back to the old directory at the end -cd \$_OLD_DIR; unset _OLD_DIR +cd $_OLD_DIR; unset _OLD_DIR From e588ef834acb5cea052e05abea20743004d9a5b0 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Tue, 5 Feb 2019 20:28:28 -0700 Subject: [PATCH 13/19] Adding changes to install script, works fine for non-interactive. Still having awk(ward) problems --- install-scripts/install_ipfix-rita.sh | 86 ++++++++++++++++----------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 21632c3..15e3749 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -118,6 +118,16 @@ do_system_tests () { return 0 } #End of do_system_tests +usage_text () { + cat >&2 < $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \\ +' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \ mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors set -e # Change dir to script dir +echo "tyring to cd to $(dirname "$BASH_SOURCE[0]")" _OLD_DIR=$(pwd); cd "$(dirname "$BASH_SOURCE[0]")"; if [[ $EUID -ne 0 ]]; then From 5aa93e85661fb4c0a632745699a223f4839cf9e7 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Wed, 6 Feb 2019 11:34:12 -0700 Subject: [PATCH 14/19] Finished the noninteractive flag debugging --- install-scripts/install_ipfix-rita.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 15e3749..a88a933 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -306,17 +306,17 @@ write_converter_conf() { RITA_MONGO_TLS_CHECK_CERT=$5 RITA_MONGO_TLS_CERT_PATH=$6 - awk -v db_root="$RITA_DATASET_DBROOT" \ - -v mongo_uri="$RITA_MONGO_URI" \ - -v mongo_auth="$RITA_MONGO_AUTH" \ - -v mongo_tls_enable="$RITA_MONGO_TLS" \ - -v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ - -v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' +awk -v db_root="$RITA_DATASET_DBROOT" \ +-v mongo_uri="$RITA_MONGO_URI" \ +-v mongo_auth="$RITA_MONGO_AUTH" \ +-v mongo_tls_enable="$RITA_MONGO_TLS" \ +-v mongo_tls_cert_check="$RITA_MONGO_TLS_CHECK_CERT" \ +-v mongo_tls_ca_path="$RITA_MONGO_TLS_CERT_PATH" ' # flag is used to determine if we are in the right scope # Unset the flag if we see "abc:" or " abc:" on a line # by itself if there are 2 or less preceding spaces -/^( )?[^ ]+:\$/{ +/^( )?[^ ]+:$/{ flag="" } @@ -375,7 +375,7 @@ flag && NF && /DBRoot:/{ 1 ' $INSTALLATION_ETC_DIR/converter/converter.yaml > $INSTALLATION_ETC_DIR/converter/converter-new.yaml && \ -mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml + mv $INSTALLATION_ETC_DIR/converter/converter-new.yaml $INSTALLATION_ETC_DIR/converter/converter.yaml } #End of write_converter_conf # Stop if there are any errors From c1f6a798b0a1ab4686dacadaf69eba3502035be4 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Wed, 13 Feb 2019 11:10:41 -0700 Subject: [PATCH 15/19] Modified script to read and write to &2, also some slight error checking if the user doesn't have a mongo connection string --- docs/Additional Info.md | 6 + install-scripts/install_ipfix-rita.sh | 179 ++++++++++++++++---------- 2 files changed, 115 insertions(+), 70 deletions(-) diff --git a/docs/Additional Info.md b/docs/Additional Info.md index cf48048..1ca84ea 100644 --- a/docs/Additional Info.md +++ b/docs/Additional Info.md @@ -8,6 +8,12 @@ code. The main script will be located at `/opt/ipfix-rita/bin/ipfix-rita`. `/etc/ipfix-rita` contains the configuration files needed to run IPFIX-RITA. +You can install IPFIX-RITA either interactively or non-interactively. To install +non-interactively run `install_ipfix-rita.sh rita address.to.rita.system`, to +install the software interactively just run `install_ipfix-rita.sh` and follow +the on screen prompts (more details later). If you want to install RITA with +a non-interactive prompt just run `install_ipfix-rita.sh rita 127.0.0.1`. + As the installer runs, it will prompt for configuration details regarding the RITA database connection and the names of the resulting datasets. Further configuration options can be set in `/etc/ipfix-rita/converter/converter.yaml` such as the CIDR diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index a88a933..f7f0e67 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -28,6 +28,33 @@ check_remote_mongo() { } #End of check_remote_mongo +strip_mongo() { + MONGO_URI_STR=$1 + + if [ `echo $MONGO_URI_STR | grep mongo` ] ; then + RITA_STR=${MONGO_URI_STR##m*/} + RITA_STR=${RITA_STR%:*} + else + RITA_STR=$MONGO_URI_STR + fi + + echo $RITA_STR +} + + +add_mongo() { + URI_STR=$1 + + if [ `echo $URI_STR | grep mongo` ] ; then + MONGO_URI=$URI_STR + else + MONGO_URI="mongodb://$URI_STR:27017" + fi + + echo $MONGO_URI +} + + can_ssh () { #Test that we can reach the target system over ssh. @@ -45,7 +72,7 @@ can_ssh () { #echo "cannot connect to $@" fi else - echo "Please supply an ssh target as a command line parameter to can_ssh" + echo "Please supply an ssh target as a command line parameter to can_ssh" >&2 fi return $success_code @@ -65,7 +92,7 @@ fail () { } #end of fail - +#TODO update this and makes sure it's called before running the full install script do_system_tests () { while [ -n "$1" ]; do if [ "$1" = "check_sse4_2" ]; then @@ -83,34 +110,34 @@ do_system_tests () { status "Installed software tests" if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then #We have apt-get , good. - sudo apt-get -qq update > /dev/null 2>&1 + apt-get -qq update > /dev/null 2>&1 inst_retcode=100000 while [ "$inst_retcode" -gt 0 ]; do - sudo apt-get -y install gdb git wget curl make realpath lsb-release rsync tar + apt-get -y install gdb git wget curl make realpath lsb-release rsync tar inst_retcode=$? if [ "$inst_retcode" -gt 0 ]; then - echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" + echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" >&2 sleep 60 fi done #TODO test IPFIX-RITA on Contos/Red Hat System #elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then # if [ ! -x /bin/yum-config-manager ]; then - # sudo yum -y install yum-utils + # yum -y install yum-utils # fi #We have yum, good. - # sudo yum -q makecache > /dev/null 2>&1 + # yum -q makecache > /dev/null 2>&1 # #FIXME - put in place a similar loop like above for apt-get - # sudo yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar + # yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar else fail "(apt-get and dpkg-query) is installed on this system" fi require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then - echo "Centos or Redhat 7 installation detected, good." + echo "Centos or Redhat 7 installation detected, good." >&2 elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then - echo "Ubuntu Linux installation detected, good." + echo "Ubuntu Linux installation detected, good." >&2 else fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" fi @@ -122,6 +149,7 @@ usage_text () { cat >&2 <&2 + echo "IPFIX-RITA needs to write to a MongoDB database controlled by RITA." >&2 + echo "By default, this installer assumes RITA and MongoDB are installed on the Docker host." >&2 + echo "In order to support this type of installation, you will need to" >&2 + echo "add the suggested Docker interface below to the list of bindIP's in /etc/mongod.conf." >&2 + echo "If needed, please do so, and restart MongoDB before continuing." >&2 + echo "Restart MongoDB with \$ sudo systemctl restart mongod.service" >&2 + echo "Note: the default configuration is not recommended. IPFIX-RITA will likely perform" >&2 + echo "better if it is installed on a machine separate from RITA/ MongoDB." >&2 + echo "" >&2 + + echo "What MongoDB URI should IPFIX-RITA use to contact the RITA database [$RITA_MONGO_URI]: " >&2 read RITA_MONGO_URI <&2 - if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then + if [ -n "$RITA_MONGO_URI" -a "$RITA_MONGO_URI" != '127.0.0.1' ]; then + rita_uri=`strip_mongo $RITA_MONGO_URI` + RITA_MONGO_URI=`add_mongo $RITA_MONGO_URI` echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 if askYN ; then echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 - read rita_user <&2 + read rita_user if [ -n "$rita_user" ]; then - rita_system="${rita_user}@${rita_system}" + rita_system="${rita_user}@${rita_uri}" fi fi + #This will check if the Rita system is accessible to our installer + #TODO, this cases issues, we have mongodb://127.0.0.1:27017 then we + # have ritauser@mongodb://127.0.0.1:27017 need to detect if they have + # mongodb and the port before trying to validate rita_system=`validated_ssh_target rita "$rita_system"` else rita_system="127.0.0.1" @@ -197,17 +232,17 @@ get_rita_data_interactive() { RITA_MONGO_AUTH="null" #instead of prompting get the RITA address and split from there to fill in our # config file - echo "" - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" - - while read && [[ ! ( "$REPLY" =~ ^[123]$ || -z "$REPLY" ) ]]; do - echo "Which authentication scheme should be used to contact the database if any? [None]" - echo "1) None" - echo "2) SCRAM-SHA-1" - echo "3) MONGODB-CR" + echo "" >&2 + echo "Which authentication scheme should be used to contact the database if any? [None]" >&2 + echo "1) None" >&2 + echo "2) SCRAM-SHA-1" >&2 + echo "3) MONGODB-CR" >&2 + + while read <&2 && [[ ! ( "$REPLY" =~ ^[123]$ || -z "$REPLY" ) ]]; do + echo "Which authentication scheme should be used to contact the database if any? [None]" >&2 + echo "1) None" >&2 + echo "2) SCRAM-SHA-1" >&2 + echo "3) MONGODB-CR" >&2 done if [ "$REPLY" = "2" ]; then @@ -219,41 +254,43 @@ get_rita_data_interactive() { RITA_MONGO_TLS="false" RITA_MONGO_TLS_CHECK_CERT="false" RITA_MONGO_TLS_CERT_PATH="null" - echo "" - read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r + echo "" >&2 + read -p "Does the MongoDB server accept TLS connections? (y/n) [n] " -r <&2 if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS="true" RITA_MONGO_TLS_CHECK_CERT="true" RITA_MONGO_TLS_CERT_PATH="null" - read -p "Would you like to provide a certificate authority? (y/n) [n] " -r + read -p "Would you like to provide a certificate authority? (y/n) [n] " -r <&2 if [[ "$REPLY" =~ ^[Yy]$ ]]; then - read -p "CA Path: " + read -p "CA Path: " <&2 RITA_MONGO_TLS_CERT_PATH="$REPLY" fi if [ "$RITA_MONGO_TLS_CERT_PATH" = "null" ]; then - read -p "Would you like to disable certificate checks? [n] " -r + read -p "Would you like to disable certificate checks? [n] " -r <&2 if [[ "$REPLY" =~ ^[Yy]$ ]]; then RITA_MONGO_TLS_CHECK_CERT="false" fi fi fi - echo "" - echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" - echo "where DBROOT consists of alphanumerics, underscores, and hyphens." + echo "" >&2 + echo "Each dataset produced with IPFIX-RITA will be named DBROOT-DATE" >&2 + echo "where DBROOT consists of alphanumerics, underscores, and hyphens." >&2 RITA_DATASET_DBROOT="IPFIX" - read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r + read -p "What would you like to set DBROOT to for this IPFIX collector? [IPFIX] " -r <&2 if [ -n "$REPLY" ]; then RITA_DATASET_DBROOT="$REPLY" fi + echo "Sending $RITA_MONGO_AUTH to the function" write_converter_conf $RITA_DATASET_DBROOT $RITA_MONGO_URI $RITA_MONGO_AUTH $RITA_MONGO_TLS $RITA_MONGO_TLS_CHECK_CERT $RITA_MONGO_TLS_CERT_PATH } #End of get_rita_data_interactive get_rita_data_noninteractive() { rita_system=$1 if [ -n "$rita_system" -a "$rita_system" != '127.0.0.1' ]; then + rita_system=`strip_mongo $rita_system` echo -n "Do you access that system with a username different than ${USER} (Y/N)" >&2 if askYN ; then echo "Enter that username now, otherwise just press enter if your account is ${USER} on the remote system too." >&2 @@ -290,7 +327,6 @@ get_rita_data_noninteractive() { #If the config specified a CA file, let's copy that if [[ $mongo_tls_ca_path != *"null" ]]; then - echo $mongo_tls_ca_path ca_file=${mongo_tls_ca_path#"CAFile: "} scp $rita_system:$ca_file $ca_file fi @@ -381,11 +417,10 @@ flag && NF && /DBRoot:/{ # Stop if there are any errors set -e # Change dir to script dir -echo "tyring to cd to $(dirname "$BASH_SOURCE[0]")" _OLD_DIR=$(pwd); cd "$(dirname "$BASH_SOURCE[0]")"; if [[ $EUID -ne 0 ]]; then - echo "This script must be run with administrator privileges." + echo "This script must be run with administrator privileges." >&2 exit 1 fi @@ -396,11 +431,13 @@ if [ "$1" = 'help' -o "$1" = '--help' ]; then exit 1 fi +#SEARCHFORTHIS while [ -n "$1" ]; do case "$1" in rita|Rita|RITA) - if [ -n "$2" ] && ! echo "$2" | egrep -iq '(^rita$|^aih$)' ; then + if [ -n "$2" ] && ! echo "$2" | egrep -iq '(^rita$)' ; then rita_system="$2" + echo "Got the rita_system" shift else rita_system='127.0.0.1' @@ -419,30 +456,31 @@ done # Install docker if needed chmod +x "install-scripts/install_docker.sh" -sudo install-scripts/install_docker.sh +install-scripts/install_docker.sh # Set by make-release -INSTALLATION_DIR="REPLACE_WITH_INSTALL_DIR" +INSTALLATION_DIR="/opt/ipfix-rita" INSTALLATION_BIN_DIR="$INSTALLATION_DIR/bin" INSTALLATION_LIB_DIR="$INSTALLATION_DIR/lib" -INSTALLATION_ETC_DIR="REPLACE_WITH_ETC_DIR" -DOCKER_IMAGES="./REPLACE_WITH_TARBALL" +INSTALLATION_ETC_DIR="/etc/ipfix-rita" +DOCKER_IMAGES="docker-images.tgz" -echo "Loading IPFIX-RITA Docker images... This may take a few minutes." +#Should we open these at the end? +echo "Loading IPFIX-RITA Docker images... This may take a few minutes." >&2 gzip -d -c ${DOCKER_IMAGES} | docker load -echo "Installing configuration files to $INSTALLATION_ETC_DIR" +echo "Installing configuration files to $INSTALLATION_ETC_DIR" >&2 SETUP_CONFIG="true" if [ ! -d "$INSTALLATION_ETC_DIR" ]; then cp -r pkg/etc "$INSTALLATION_ETC_DIR" else # TODO: set up migration - echo "Existing configuration found. Skipping..." + echo "Existing configuration found. Skipping..." >&2 SETUP_CONFIG="false" fi -echo "Installing ipfix-rita in $INSTALLATION_DIR" +echo "Installing ipfix-rita in $INSTALLATION_DIR" >&2 if [ -d "$INSTALLATION_DIR" ]; then rm -rf "$INSTALLATION_DIR" @@ -460,7 +498,7 @@ RECV_BUFF_SIZE=$(sysctl -n net.core.rmem_max) RECV_BUFF_OPT_SIZE="$((1024*1024*64))" if [ "$RECV_BUFF_SIZE" -lt "$RECV_BUFF_OPT_SIZE" ]; then sysctl -w net.core.rmem_max=$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf + echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf >&2 fi "$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start @@ -474,26 +512,26 @@ if [ "$SETUP_CONFIG" = "true" ]; then fi #We have written to config - echo "" - echo "Your settings have been saved to $INSTALLATION_ETC_DIR/converter/converter.yaml" - echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" - echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" - echo "in $INSTALLATION_ETC_DIR/converter/converter.yaml." + echo "" >&2 + echo "Your settings have been saved to $INSTALLATION_ETC_DIR/converter/converter.yaml" >&2 + echo "Note: By default IPFIX-RITA, considers all Class A, B, and C IPv4 networks" >&2 + echo "as local networks. If this is not the case, please edit the list 'LocalNetworks'" >&2 + echo "in $INSTALLATION_ETC_DIR/converter/converter.yaml." >&2 fi -echo "" -echo "IPFIX-RITA will run at start up unless the system has been stopped." -echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." -echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." -echo "To view the system logs, run 'ipfix-rita logs -f'." +echo "" >&2 +echo "IPFIX-RITA will run at start up unless the system has been stopped." >&2 +echo "In order to stop IPFIX-RITA, run 'ipfix-rita stop'." >&2 +echo "To restart IPFIX-RITA, run 'ipfix-rita up -d'." >&2 +echo "To view the system logs, run 'ipfix-rita logs -f'." >&2 echo "" -echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to $INSTALLATION_BIN_DIR/ipfix-rita." +echo "Adding a symbolic link from /usr/local/bin/ipfix-rita to $INSTALLATION_BIN_DIR/ipfix-rita." >&2 ln -fs "$INSTALLATION_BIN_DIR/ipfix-rita" /usr/local/bin/ipfix-rita -echo "" -echo "Starting IPFIX-RITA..." +echo "" >&2 +echo "Starting IPFIX-RITA..." >&2 "$INSTALLATION_BIN_DIR/ipfix-rita" up -d @@ -501,3 +539,4 @@ echo "The IPFIX-RITA installer has finished." # Change back to the old directory at the end cd $_OLD_DIR; unset _OLD_DIR + From 2edc981d3a7bfa7ecbbe88b8c08226d19e7786e3 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Thu, 14 Feb 2019 12:02:50 -0700 Subject: [PATCH 16/19] Adding a directory structure to start of the make-release script so user's understand how everything is packaged in a release. --- dev-scripts/make-release.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index 09efde7..00e4aa4 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -1,5 +1,33 @@ #!/usr/bin/env bash +#This script creates a tgz file with the following structure and contents. +#ipfix-rita +#├── docker-images.tgz +#├── docs +#|   ├── Additional Info.md +#│   ├── Developer Notes.md +#│   ├── Generating Data.md +#│   └── Router Support.md +#├── install_ipfix-rita.sh +#├── install-scripts +#│   ├── helpers +#│   │   ├── check_docker-compose.sh +#│   │   └── check_docker.sh +#│   └── install_docker.sh +#├── pkg +#│   ├── bin +#│   │   └── ipfix-rita +#│   ├── etc +#│   │   └── converter +#│   │   └── converter.yaml +#│   └── lib +#│   └── docker-compose +#│   ├── main.yaml +#│   ├── no-rotate.yaml +#│   └── xpack.yaml +#└── README.md +# + # Change dir to script dir pushd "$(dirname "$BASH_SOURCE[0]")" > /dev/null From b137746c737761aa604c7c62327e0578f638e8a5 Mon Sep 17 00:00:00 2001 From: Samuel Carroll Date: Thu, 14 Feb 2019 13:17:53 -0700 Subject: [PATCH 17/19] Commenting out the system check for now, will add back in --- install-scripts/install_ipfix-rita.sh | 79 +++++++++++++-------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index f7f0e67..7a2ccb7 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -93,33 +93,33 @@ fail () { #TODO update this and makes sure it's called before running the full install script -do_system_tests () { - while [ -n "$1" ]; do - if [ "$1" = "check_sse4_2" ]; then - status "Hardware tests" - require_file /proc/cpuinfo || fail "Missing /proc/cpuinfo - is this a Linux system?" - fi - shift - done - - for one_dir in $HOME / /usr ; do +# do_system_tests () { +# while [ -n "$1" ]; do +# if [ "$1" = "check_sse4_2" ]; then +# status "Hardware tests" +# require_file /proc/cpuinfo || fail "Missing /proc/cpuinfo - is this a Linux system?" +# fi +# shift +# done + +# for one_dir in $HOME / /usr ; do #"tr" in next command removes the linefeed at the end of the number of free megabytes (and any other whitespace) - [ `df -h $one_dir --output=avail --block-size=1048576 | grep -v '^Avail' | tr -dc '[0-9]'` -gt 5000 ] || fail "${one_dir} has less than 5GB free space" - done +# [ `df -h $one_dir --output=avail --block-size=1048576 | grep -v '^Avail' | tr -dc '[0-9]'` -gt 5000 ] || fail "${one_dir} has less than 5GB free space" +# done - status "Installed software tests" - if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then +# status "Installed software tests" +# if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then #We have apt-get , good. - apt-get -qq update > /dev/null 2>&1 - inst_retcode=100000 - while [ "$inst_retcode" -gt 0 ]; do - apt-get -y install gdb git wget curl make realpath lsb-release rsync tar - inst_retcode=$? - if [ "$inst_retcode" -gt 0 ]; then - echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" >&2 - sleep 60 - fi - done +# apt-get -qq update > /dev/null 2>&1 +# inst_retcode=100000 +# while [ "$inst_retcode" -gt 0 ]; do +# apt-get -y install gdb git wget curl make realpath lsb-release rsync tar +# inst_retcode=$? +# if [ "$inst_retcode" -gt 0 ]; then +# echo "Error installing packages, perhaps because a system update is running; will wait 60 seconds and try again" >&2 +# sleep 60 +# fi +# done #TODO test IPFIX-RITA on Contos/Red Hat System #elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then # if [ ! -x /bin/yum-config-manager ]; then @@ -129,21 +129,21 @@ do_system_tests () { # yum -q makecache > /dev/null 2>&1 # #FIXME - put in place a similar loop like above for apt-get # yum -y install gdb git wget curl make coreutils coreutils redhat-lsb-core rsync tar - else - fail "(apt-get and dpkg-query) is installed on this system" - fi - require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" - - if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then - echo "Centos or Redhat 7 installation detected, good." >&2 - elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then - echo "Ubuntu Linux installation detected, good." >&2 - else - fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" - fi - - return 0 -} #End of do_system_tests +# else +# fail "(apt-get and dpkg-query) is installed on this system" +# fi +# require_util awk cat cp curl date egrep gdb getent git grep ip lsb_release make mkdir mv printf rm rsync sed sleep tar tr wc wget || fail "A needed tool is missing" + +# if [ -s /etc/redhat-release ] && grep -iq 'release 7' /etc/redhat-release ; then +# echo "Centos or Redhat 7 installation detected, good." >&2 +# elif grep -iq '^DISTRIB_ID *= *Ubuntu' /etc/lsb-release ; then +# echo "Ubuntu Linux installation detected, good." >&2 +# else +# fail "This system does not appear to be a Centos/RHEL 7 or Ubuntu Linux system" +# fi + +# return 0 +# } #End of do_system_tests usage_text () { cat >&2 < Date: Thu, 14 Feb 2019 17:00:06 -0700 Subject: [PATCH 18/19] Fix bug where config value was sent to stderr. Ensure sysctl command output is sent to stderr --- install-scripts/install_ipfix-rita.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-scripts/install_ipfix-rita.sh b/install-scripts/install_ipfix-rita.sh index 7a2ccb7..89ec9a4 100755 --- a/install-scripts/install_ipfix-rita.sh +++ b/install-scripts/install_ipfix-rita.sh @@ -497,8 +497,8 @@ cp -r ./pkg/lib "$INSTALLATION_LIB_DIR" RECV_BUFF_SIZE=$(sysctl -n net.core.rmem_max) RECV_BUFF_OPT_SIZE="$((1024*1024*64))" if [ "$RECV_BUFF_SIZE" -lt "$RECV_BUFF_OPT_SIZE" ]; then - sysctl -w net.core.rmem_max=$RECV_BUFF_OPT_SIZE - echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf >&2 + sysctl -w net.core.rmem_max=$RECV_BUFF_OPT_SIZE >&2 + echo "net.core.rmem_max=$RECV_BUFF_OPT_SIZE" >> /etc/sysctl.conf fi "$INSTALLATION_BIN_DIR/ipfix-rita" up --no-start From c12e04a4c4355678a6ea045549e2c7a68b668668 Mon Sep 17 00:00:00 2001 From: Logan L Date: Tue, 12 Mar 2019 14:22:44 -0600 Subject: [PATCH 19/19] Pull mongo image in case the user running make-release doesn't have it installed --- dev-scripts/make-release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-scripts/make-release.sh b/dev-scripts/make-release.sh index 00e4aa4..6ea4030 100755 --- a/dev-scripts/make-release.sh +++ b/dev-scripts/make-release.sh @@ -53,6 +53,7 @@ IN_DEV_MAIN_COMPOSE_FILE="$IN_DEV_COMPOSE_DIR/main.yaml" ################################################################################ echo "Building Docker images" COMPOSE_FILE="$IN_DEV_MAIN_COMPOSE_FILE" docker-compose build +docker pull mongo:3.6 ################################################################################ TMP_DIR=`mktemp -d -q "/tmp/IPFIX-RITA.XXXXXXXX"