cxy_sdn

#!/bin/bash
# cxy-k8s-network-plugin

# This plugin act as a client of cxy-sdn. The heavy lifting work is done by cxy-sdn server. They are communicating via HTTP

# According to https://github.com/GoogleCloudPlatform/kubernetes/pull/5069
# This plugin need to implement two specific hooks
# 1 After pod infra container is launched (before other containers of the pod are launched)
# 2 Before pod infra container is deleted

set -e

usage() {
    cat << EOF
NAME:
    cxy_sdn - Install, Configure and Run cxy-sdn

VERSION:
    10.0

USAGE:
    $0 <options> <command> [command_options] [arguments...]

COMMANDS:
    help
            Help and usage

    install [unattended] [nopowerstrip]
            Install cxy-sdn (installs docker and ovs)

    uninstall
            Remove cxy-sdn installation

    clean
            Remove cxy-sdn installation and dependencies (docker and openvswitch)

    deps
            Show cxy-sdn dependencies

    agent {stop|start|restart|logs}
            Start/Stop/Restart the cxy-sdn container or show its logs

    info [container_id]
            Show cxy-sdn info for all containers, or for a given container_id

    run [-n foo] <docker_run_args>
            Run a container and optionally specify which network to attach to

    start <container_id>
            Start a <container_id>

    stop <container_id>
            Stop the <container_id>

    rm <container_id>
            Remove the <container_id>

    attach <container_id>
            Attach to the <container_id>

    cluster bind <interface>
            Bind clustering to a specific interface

    cluster join <address>
            Join the cluster at the specified address

    cluster leave
            Leave the cluster

    network list
            List all created networks

    network info <name>
            Display information about a given network

    network create <name> [cidr]
            Create a network

    network delete <name> [cidr]
            Delete a network

    network agent start
            Starts an existing cxy-sdn image if it is not already running

    network agent stop
            Stops a running cxy-sdn image. This will not delete the local image

    init
            just init for k8s

    setup   namespace pod cid
            setup after the infra pod is created

    teardown namespace pod cid
            teardown before the infra pod is killed

    status namespace pod cid
            get pod ip(aka pod infra container ip)

    setQoS cid bw delay
           set up container's egress bandwidth and latency whose container id is CID
    

EOF
}

# Utility function to test if a command exists
command_exists() {
    hash $@ 2>/dev/null
}

# Colorized Command Output
log_info() {
    printf "\033[0;36m$@\033[0m\n"
}

log_notice() {
    printf "\033[0;32m$@\033[0m\n"
}

log_warn() {
    printf "\033[0;33m$@\033[0m\n"
}

log_error() {
    printf "\033[0;35m$@\033[0m\n"
}

log_fatal() {
    printf "\033[0;31m$@\033[0m\n"
}

log_debug() {
    printf "\033[0;37m$@\033[0m\n"
}

log_step() {
    log_info "-----> $@"
}

indent() {
    sed -u "s/^/           /"
}

get_status() {
    OS="NOT_LINUX"
    RELEASE="NOT_LINUX"
    CODENAME="NOT_LINUX"
    ARCH=$(uname -m)

    if [ "$ARCH" = "x86_64" ]; then
        ARCH="amd64";
    fi

    if [ "$ARCH" = "i686" ]; then
        ARCH="i386";
    fi

    if command_exists lsb_release; then
        OS=$(lsb_release -is)
        RELEASE=$(lsb_release -rs)
        CODENAME=$(lsb_release -cs)
    elif [ -f /etc/debian_version ]; then
        OS="Debian"
        RELEASE="UNDETECTED"
        CODENAME="UNDETECTED"
    elif [ -f /etc/redhat-release ]; then
        OS="RedHat"
        RELEASE="UNDETECTED"
        CODENAME="UNDETECTED"
    fi

    DOCKER_SVER="NOT_INSTALLED"
    DOCKER_CVER="NOT_INSTALLED"
    if command_exists docker || command_exists lxc-docker; then
        DOCKER_SVER=$(docker version | grep "Server version:" |  awk '{ print $3 }')
        DOCKER_CVER=$(docker version | grep "Client API version:" |  awk '{ print $4 }')
    fi

    OVS_SVER="NOT_INSTALLED"
    if command_exists ovsdb-server && command_exists ovs-vswitchd ; then
        OVS_SVER=$(ovs-appctl -V | grep "ovs-" |  awk '{ print $4 }')
    fi
}

deps() {
    echo "cxy-sdn  Docker Host Requirements:"
    echo ".. Open vSwitch Environment:"
    echo ".... Archicture:              amd64 or i386"
    echo "....   Current:               $ARCH"
    echo ".... Operating System:         Ubuntu, Debian and Fedora"
    echo "....   Current:               $lsb_dist"
    echo ".... Open vSwitch Version:     2.3.1"
    echo "....   Current:               $OVS_SVER"
    echo ".. Docker Environment:"
    echo ".... Docker Server Version:   1.4 or higher"
    echo "....   Current:               $DOCKER_SVER"
    echo ".... Docker Client Version:   1.16 or higher"
    echo "....   Current:               $DOCKER_CVER"
}

kernel_opts(){
    log_step "Setting Linux Kernel Options"
    if [ $(cat /etc/sysctl.conf | grep icmp_echo_ignore_broadcasts) ]; then
        sed -i 's/^#\?net\.ipv4\.icmp_echo_ignore_broadcasts.*$/net\.ipv4\.icmp_echo_ignore_broadcasts=0/g' /etc/sysctl.conf
    else
        echo 'net.ipv4.icmp_echo_ignore_broadcasts=0' >> /etc/sysctl.conf
    fi

    if [ $(cat /etc/sysctl.conf | grep ip_forward ) ]; then
        sed -i 's/^#\?net\.ipv4\.ip_forward.*$/net\.ipv4\.ip_forward=1/g' /etc/sysctl.conf
    else
        echo 'net.ipv4.ip_forward=1' >> /etc/sysctl.conf
    fi
    sysctl -p | indent
}

pkg_update() {
    log_step "Ensuring Package Repositories are up to date"
    $pkg update > /dev/null
}

install_curl() {
    if command_exists curl; then
        log_step "Curl already installed!"
    else
        log_step "Installing Curl"
        $pkg install curl | indent
    fi
}

install_ovs() {
    if command_exists ovsdb-server && command_exists ovs-vswitchd ; then
        log_step "Open vSwitch already installed!"
    else
        if command_exists getenforce && [ -z $(getenforce | grep -E 'Enforcing|Permissive') ]  ; then
            log_step "Checking Open vSwitch dependencies.."
            $pkg install $policy
            sudo semodule -d openvswitch  2>/dev/null || true
        fi
        log_step "Installing Open vSwitch.."
        $pkg install $ovs | indent
    fi

    # Make sure the processes are started
    case "$lsb_dist" in
        debian|ubuntu)
            if $(service openvswitch-switch status | grep "stop"); then
                service openvswitch-switch start
            fi
            ;;
        fedora)
            systemctl start openvswitch.service
            ;;
    esac

    sleep 1
    log_step "Setting OVSDB Listener" #important!
    ovs-vsctl set-manager ptcp:6640
}

remove_ovs() {
    log_step "Removing existing Open vSwitch packages..."
    $pkg remove $ovs
}

install_docker() {
    if command_exists docker; then
        log_step "Docker already installed!"
    else
        log_step "Installing Docker..."
        case $lsb_dist in
            fedora)
                $pkg remove docker > /dev/null
                ;;
        esac

        if command_exists curl; then
            curl -sSL https://get.docker.com/ | sh
        elif command_exists wget; then
            wget -qO- https://get.docker.com/ | sh
        fi
    fi

    case $lsb_dist in
        debian|ubuntu)
            if [ -f /etc/init.d/docker ]; then
                if $(service docker status | grep "stop"); then
                    service docker start
                fi
            else
                if $(service docker.io status | grep "stop"); then
                    service docker.io start
                fi
            fi
            ;;
        fedora)
            systemctl start docker.service
            ;;
    esac
}

start_cxy-sdn() {
    log_step "Starting the cxy-sdn container"

    if [ -n "$(docker ps | grep registry.aliyuncs.com/wizardcxy/cxy-sdn | awk '{ print $1 }')" ]; then
        log_fatal "A cxy-sdn container is already running"
        return 1
    fi

    flags="--iface=eth0"

    while true; do
        read -p "Is this a server node in the cluster? (y/n)" yn
        case $yn in
            [Yy]* )
                flags="$flags --server=true"
                read -p "what is the server number: " num
                flags="$flags --expectedServerNum=$num"
                break
                ;;
            [Nn]* )
                break
                ;;
            * )
                echo "Please answer yes or no."
                ;;
        esac
    done

    cid=$(docker run -itd --privileged=true \
	-v /var/run/docker.sock:/var/run/docker.sock \
    -v /usr/bin/ovs-vsctl:/usr/bin/ovs-vsctl -v /var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock \
	-v /usr/bin/docker:/usr/bin/docker -v /proc:/hostproc -e PROCFS=/hostproc \
	--net=host registry.aliyuncs.com/wizardcxy/cxy-sdn /cxy-sdn $flags)

    if [ -n "$cid" ]; then
        log_info "A cxy-sdn container was started" | indent
    else
        log_fatal "Error starting the cxy-sdn container"
        exit 1
    fi

    mkdir -p /var/run/cxy-sdn
    echo $cid > /var/run/cxy-sdn/cid
}

stop_cxy-sdn() {
    log_step "Stopping cxy-sdn agent"
    if ! command_exists docker; then
        log_fatal "Docker is not installed, please run \"./cxy-sdn install\""
        exit 1
    fi

    for CID in $(docker ps | grep registry.aliyuncs.com/wizardcxy/cxy-sdn | awk '{ print $1; }'); do
        log_info "Stopping the cxy-sdn container $CID" | indent
        docker kill -s=15 ${CID} > /dev/null
    done

    if [ -z $(docker ps | grep registry.aliyuncs.com/wizardcxy/cxy-sdn | awk '{ print $1 }') ]; then
        log_info "All cxy-sdn agent containers are stopped. Please run \"cxy-sdn agent start\" to start them again"
    fi
}

remove_cxy-sdn() {
    log_step "remove cxy-sdn Agent"
    if ! command_exists docker; then
        log_fatal "Docker is not installed"
        exit 1
    fi

    for CID in $(docker ps -a | grep registry.aliyuncs.com/wizardcxy/cxy-sdn | awk '{ print $1; }'); do
        log_info "Stopping cxy-sdn container: $CID" | indent
        docker stop $CID > /dev/null
        sleep 1
        log_info "Removing cxy-sdn container: $CID" | indent
        docker rm $CID > /dev/null
    done

    log_info "cxy-sdn container deleted" | indent
}

logs() {
    if [ ! -f /var/run/cxy-sdn/cid ] || [ -z $(cat /var/run/cxy-sdn/cid) ]; then
        log_fatal "cxy-sdn container is not running"
        exit 1
    fi
    docker logs $@ $(cat /var/run/cxy-sdn/cid)
}

info() {
    if [ -z "$1" ]; then
        curl -s -X GET http://localhost:8888/connections | python -m json.tool
    else
        containerId=$(docker ps -a --no-trunc=true | grep $1 | awk {' print $1'})
        if [ -z "$containerId" ]; then
            log_fatal "Could not find a Container with Id : $1"
        else
            curl -s -X GET http://localhost:8888/connection/$containerId | python -m json.tool
        fi
    fi
}

container_run() {
    network=""
    requestIp=""
    if [ $1 == '-n' ]; then
        network=$2
        shift 2
    fi

    if [ $1 == "-z" ]; then
        requestIp=$2
        shift 2
    fi

    attach="false"
    if [ -z "$(echo "$@" | grep -e '-[a-zA-Z]*d[a-zA-Z]*\s')" ]; then
        attach="true"
    fi

    if [ "$attach" = "false" ]; then
         cid=$(docker run --net=none $@)
    else
         cid=$(docker run --net=none -d $@)
    fi

    cPid=$(docker inspect --format='{{ .State.Pid }}' $cid)
    cName=$(docker inspect --format='{{ .Name }}' $cid)

    json=$(curl -s -X POST http://localhost:8888/connection -d "{ \"containerID\": \"$cid\", \"containerName\": \"$cName\", \"requestIP\": \"$requestIp\", \"containerPID\": \"$cPid\", \"network\": \"$network\" }")
    result=$(echo $json | sed 's/[,{}]/\n/g' | sed 's/^".*":"\(.*\)"/\1/g' | awk -v RS="" '{ print $7, $8, $9, $10, $11 }')

    if [ "$attach" = "false" ]; then
        echo $cid
    else
        docker attach $cid
    fi
}

container_stop() {
    docker stop $@
    find -L /var/run/netns -type l -delete
}

container_start() {
    docker start $1 > /dev/null
    cid=$(docker ps --no-trunc=true | grep $1 | awk {' print $1'})
    cPid=$(docker inspect --format='{{ .State.Pid }}' $cid)
    cName=$(docker inspect --format='{{ .Name }}' $cid)

    json=$(curl -s -X GET http://localhost:8888/connection/$cid)
    result=$(echo $json | sed 's/[,{}]/\n/g' | sed 's/^".*":"\(.*\)"/\1/g' | awk -v RS="" '{ print $7, $8, $9, $10, $11 }')

    attach $result $cPid

    # ToDo: PUT new container info to the api

    echo $cid

}

container_delete() {
    cid=$(docker ps -a --no-trunc=true | grep $1 | awk {' print $1'})
    docker rm $@
    curl -s -X DELETE http://localhost:8888/connection/$cid
    sleep 1
}

cluster_join(){
    log_info "Requesting cxy-sdn to join the cluster at $*"
    curl -s -X POST http://localhost:8888/cluster/join?address=$*
}

cluster_leave(){
    log_info "Requesting cxy-sdn to leave cluster"
    curl -s -X POST http://localhost:8888/cluster/leave
}

network_list() {
    curl -s -X GET http://localhost:8888/networks | python -m json.tool
}

network_info() {
    curl -s -X GET http://localhost:8888/network/$1 | python -m json.tool
}

network_create() #name
                 #cidr
{
    #ToDo: Check CIDR is valid
    curl -s -X POST http://localhost:8888/network -d "{ \"name\": \"$1\", \"subnet\": \"$2\" }" | python -m json.tool

}

network_delete() {
    curl -s -X DELETE http://localhost:8888/network/$@
}

# Run as root only
if [ "$(id -u)" != "0" ]; then
    log_fatal "Please run as root"
    exit 1
fi

# perform some very rudimentary platform detection
lsb_dist=''
if command_exists lsb_release; then
    lsb_dist="$(lsb_release -si)"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/lsb-release ]; then
    lsb_dist="$(. /etc/lsb-release && echo "$DISTRIB_ID")"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/debian_version ]; then
    lsb_dist='debian'
fi
if [ -z "$lsb_dist" ] && [ -r /etc/fedora-release ]; then
    lsb_dist='fedora'
fi
if [ -z "$lsb_dist" ] && [ -r /etc/os-release ]; then
    lsb_dist="$(. /etc/os-release && echo "$ID")"
fi

lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"

if [ -z lsb_dist ]; then
    log_fatal "Operating System could not be detected"
    exit 1
fi

if [ -z "$(echo "$lsb_dist" | grep -E 'ubuntu|debian|fedora')" ]; then
    log_fatal "Operating System $lsb_dist is not yet supported. Please contact support@cxy-sdn.io"
    exit 1
fi

case "$lsb_dist" in
    debian|ubuntu)
        pkg="apt-get -q -y --force-yes"
        ovs="openvswitch-switch"
        policy="policycoreutils"
        ;;
    fedora)
        pkg="yum -q -y"
        ovs="openvswitch"
        policy="policycoreutils-python"
        ;;
esac

while getopts ":D" opt; do
  case $opt in
    D)
      DEBUG="true"
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      ;;
  esac
done
shift $((OPTIND-1))

if [ "$DEBUG" = "true" ]; then
    set -x
fi

case "$1" in
    init)
        echo "hello in cxy-sdn init"
        ;;
    setup)
        echo "set up network"
        # $0 is command name, $2 is pod namespace used as network name for cxy-sdn
        # $4 is container id
        cPid=$(docker inspect --format='{{ .State.Pid }}' $4)
        cName=$(docker inspect --format='{{ .Name }}' $4)
        json=$(curl -s -X POST http://localhost:8888/connection -d "{ \"containerID\": \"$4\", \"containerName\": \"$cName\", \"requestIP\": \"$requestIp\", \"containerPID\": \"$cPid\", \"network\": \"$2\" }")
        result=$(echo $json | sed 's/[,{}]/\n/g' | sed 's/^".*":"\(.*\)"/\1/g' | awk -v RS="" '{ print $7, $8, $9, $10, $11 }')
        ;;
    teardown)
        echo "teardown pod"
        # $0 is command name, $2 is pod namespace used as network name for cxy-sdn
        # $4 is container id
        cid=$(docker ps -a --no-trunc=true | grep $4 | awk {' print $1'})
        curl -s -X DELETE http://localhost:8888/connection/$cid
        sleep 1
        ;;
    status)
        # $0 is command name, $2 is pod namespace used as network name for cxy-sdn
        # $4 is container id
        containerId=$(docker ps -a --no-trunc=true | grep $4 | awk {' print $1'})
        if [ -z "$containerId" ]; then
            log_fatal "Could not find a Container with Id : $containerId"
        else
            json=$(curl -s -X GET http://localhost:8888/connection/$containerId | python -m json.tool | grep ip)
            ip=$(echo $json | sed 's/[,{}]/\n/g' | awk  -v RS="" '{print $2}')
            echo ""{ \"kind\": \"PodNetworkStatus\", \"apiVersion\": \"v1beta1\", \"ip\": $ip }""
        fi
        ;;
    help)
        usage
        ;;
    install)
        shift
        log_notice "Installing cxy-sdn..."
        kernel_opts
        #pkg_update
        install_curl
        install_ovs
        install_docker
        start_cxy-sdn $@
        log_notice "Done!!!"
        ;;
    uninstall)
        log_notice "Uninstalling cxy-sdn..."
        stop_cxy-sdn
        remove_cxy-sdn
        log_notice "Done!!!"
        ;;
    clean)
        log_notice "Removing cxy-sdn and all its dependencies..."
        get_status
        remove_ovs
        stop_cxy-sdn
        remove_cxy-sdn
        log_notice "Done!!!"
        ;;
    deps)
        get_status
        deps
        ;;
    info)
        shift
        info $@
        ;;
    run)
        shift
        container_run $@
        ;;
    stop)
        shift
        container_stop $@
        ;;
    start)
        shift
        container_start $@
        ;;
    rm)
        shift
        container_delete $@
        ;;
    attach)
        shift
        docker attach $@
        ;;
    cluster)
        shift
        case $1 in
    	    join)
    		shift
    		cluster_join $@
    		;;
            leave)
	    	cluster_leave
    		;;
    	    *)
            log_fatal "Unknown Command"
            usage
            exit 1
        esac
        ;;
    network)
        shift
        case "$1" in
            list)
                network_list
                ;;
            info)
                shift
                network_info $@
                ;;
            create)
                shift
                network_create $@
                ;;
            delete)
                shift
                network_delete $@
                ;;
            *)
                log_fatal "Unknown Command"
                usage
                exit 1
        esac
        ;;
    agent)
        shift 1
        case "$1" in
            start)
                start_cxy-sdn
                ;;
            stop)
                stop_cxy-sdn
                ;;
            restart)
                stop_cxy-sdn
                start_cxy-sdn
                ;;
            logs)
                shift 1
                logs $@
                ;;
            *)
                log_fatal "\"cxy-sdn agent\" {stop|start|restart|logs}"
                exit 1
                ;;
        esac
        ;;
    qos)
        shift
        # QoS setting
        case "$1" in
            add)
                shift
                curl -X POST -d "bw=${2}&delay=${3}" http://127.0.0.1:8888/qos/${1}
                ;;
            update)
                shift
                curl -X PUT -d "bw=${2}&delay=${3}" http://127.0.0.1:8888/qos/${1}
                ;;
            *)
            log_fatal "\"cxy-sdn qos\" {add|update}"
            exit 1
            ;;
        esac
    ;;
    *)
    usage
    exit 1
    ;;
esac