From 3faa21f4e76e13b99e4eafc1b2c84e432106ae1a Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Tue, 19 Jan 2021 17:42:30 +1300 Subject: [PATCH 1/2] Allow starting pipette with docker-compose. Simplify config. --- .env | 40 ++++++++++++++++++++++++ .github/workflows/docker.yml | 6 +++- .github/workflows/test.yml | 4 +-- CHANGELOG.md | 6 +++- Dockerfile => Dockerfile.pipette | 0 Dockerfile.pipetteconf | 9 ++++++ README.md | 20 ++++++------ configureovs.sh | 34 +++++++++++++------- docker-compose.yml | 53 ++++++++++++++++++++++++++++++++ pipetteconf.sh | 43 ++------------------------ runpipette.sh | 23 +++----------- shutdownpipette.sh | 8 ----- 12 files changed, 155 insertions(+), 91 deletions(-) create mode 100755 .env rename Dockerfile => Dockerfile.pipette (100%) create mode 100644 Dockerfile.pipetteconf create mode 100644 docker-compose.yml diff --git a/.env b/.env new file mode 100755 index 0000000..2ced6fb --- /dev/null +++ b/.env @@ -0,0 +1,40 @@ +# pipette configuration + +# NOTE: system management of all pipette's interfaces should be disabled. +# e.g. add denyinterfaces COPROINT, denyinterfaces BR, etc all to /etc/dhcpcd.conf + +# interface connected to FAUCET coprocessor port. +COPROINT=lo +# FAUCET VLANS where fake services will appear (space separated). +VLANS=2 +# addresses fake services will be run on (will be proxied from real IPs) +# At the moment must be /16, and real network must be /24 (for NAT to work). +# space separated. +# There must be the same number of IPs in NFVIPs, as there are VLANs. +NFVIPS=10.10.0.1/16 +# IPv6 is also supported - must be a /64, the real network must be a /96. +# NFVIPS=fc04::1/64 +# interface that will be created for fake services to run on. +FAKEINT=fake0 + +## +# Optional config +## + +# Reserved MAC addresses for fake services to use to talk to clients. +FAKESERVERMAC=0e:00:00:00:00:66 +FAKECLIENTMAC=0e:00:00:00:00:67 +# OVS bridge name +BR=copro0 +# pipette OF port +OF=6699 +# OF port number for interface facing coprocessor +COPROPORT=1 +# OF port number for fake interface +FAKEPORT=2 +# Flag to record and location to dump pcaps +RECORD=0 +PCAP_LOCATION=./pcaps + +#pipette temporary directory +PIPETTE_TEMP_DIR=/tmp/pipette diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3f6e7fa..54ff6b2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,5 +41,9 @@ jobs: docker buildx build \ --platform linux/amd64,linux/arm/v7,linux/arm64 \ --push \ - -t iqtlabs/pipette:${{ steps.change_version.outputs.VERSION }} . + -t iqtlabs/pipette:${{ steps.change_version.outputs.VERSION }} -f Dockerfile.pipette . && \ + docker buildx build \ + --platform linux/amd64,linux/arm/v7,linux/arm64 \ + --push \ + -t iqtlabs/pipette:${{ steps.change_version.outputs.VERSION }} -f Dockerfile.pipetteconf . if: github.repository == 'iqtlabs/pipette' && github.event_name == 'push' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14303ef..cc7d174 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - name: test run: | sudo apt-get update && \ - sudo apt-get install -y python3-dev python3-setuptools openvswitch-switch wget && \ + sudo apt-get install -y python3-dev python3-setuptools openvswitch-switch wget docker-compose && \ pip3 install -U pip && \ pip3 install codecov && \ pip3 install -U -r test-requirements.txt && \ @@ -32,4 +32,4 @@ jobs: if: github.repository == 'iqtlabs/pipette' - name: docker-test run: | - docker build -f Dockerfile -t iqtlabs/pipette . + docker-compose build diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c152e..433d07e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -### [1.3](https://github.com/IQTLabs/pipette/compare/v1.1...v1.2) (2021-01-14) +### [1.4](https://github.com/IQTLabs/pipette/compare/v1.1...v1.4) (2021-01-20) + +Allow starting pipette from docker-compose, simplify config. + +### [1.3](https://github.com/IQTLabs/pipette/compare/v1.1...v1.3) (2021-01-14) Fix shell var quoting Fix Docker build diff --git a/Dockerfile b/Dockerfile.pipette similarity index 100% rename from Dockerfile rename to Dockerfile.pipette diff --git a/Dockerfile.pipetteconf b/Dockerfile.pipetteconf new file mode 100644 index 0000000..4f6a739 --- /dev/null +++ b/Dockerfile.pipetteconf @@ -0,0 +1,9 @@ +FROM ubuntu:20.04 + +RUN apt-get update && \ + apt-get -y --no-install-recommends install openvswitch-common openvswitch-switch iproute2 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY configureovs.sh /configureovs.sh +ENTRYPOINT ["/configureovs.sh"] diff --git a/README.md b/README.md index a7d418f..f85751a 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,22 @@ Pipette is a tool that allows users to multiplex SDN coprocessing by implementing transparent L3 NAT. Pipette does this by creating a virtual network behind your coprocessor port and then acting as the SDN controller of that network. Packets are seamlessly switched to their appropriate destination using [Ryu](https://osrg.github.io/ryu/). ## Usage + +*NOTE: Running pipette outside of Docker, is deprecated and will be removed in a future release.* + + 1. Edit configuration in `.env`, as below (note `.env` is used even when not using Docker). + 1. If using Docker, start pipette with `docker-compose up -d`. If not using Docker, start pipette with `./runpipette.sh.` + 1. Start fake services listening on the NFVIP address assigned to the fake interface (Eg, IP of `fake0.2` for VLAN 2 - pipette manages this interface and assigns the NFVIP). Fake services do not have to be in Docker. + 1. When finished, `docker-compose down` if using Docker, or `./shutdownpipette.sh` + ### Configuration #### Required 1. COPROINT - the interface that will receive coprocessed packets - 1. NFVIPS - IPs to send coprocessed packets to. Must be a /16 - 1. VLANS - Space delimitted list of vlans to coprocess from, must match a vlan in ACL rule - 1. FAKEINT - interface created for fake services to run on - 1. DFILE - Dockerfile to use to run pipette. should be set based on hardware use + 1. VLANS - Space delimitted list of VLANs to coprocess from, must match a VLAN in FAUCET ACL rule + 1. NFVIPS - IPs to send coprocessed packets to. Must be a /16 if IPv4, or /96 if IPv6. There must be the same number of IPs in the list as VLANs. #### Optional + 1. FAKEINT - name of interface created for fake services to run on (default fake0) 1. FAKESERVERMAC - MAC to be assigned to the coprocessing server 1. FAKECLIENTMAC - MAC to be assigned to the coprocessing client 1. BR - name of OVS bridge to be created @@ -23,8 +30,3 @@ Pipette is a tool that allows users to multiplex SDN coprocessing by implementin 1. RECORD - 0 to not store pcaps passing through `$COPROINT`, anything else to store them 1. PCAP_LOCATION - filename of store pcaps 1. PIPETTE_TEMP_DIR temp directory to store process info -Most of the above can can be overridden by passing appropriate flags to the startup script. For more details run `./runpipette.sh --help` for more details - -### Starting Pipette -1. Run the Shell script using `./runpipette.sh`. By default this will run Pipette in a docker container, use the `--no-docker` option to run it natively. -1. Start any coprocessing services. It is important to ensure that the services are bound to one of the IPs containted in `$NFVIPS`. If using Docker besure to start the container using the `-p ::` option diff --git a/configureovs.sh b/configureovs.sh index bcdf0ca..0ff8709 100755 --- a/configureovs.sh +++ b/configureovs.sh @@ -1,22 +1,29 @@ #!/bin/bash +while ovs-vsctl show; [ $? -ne 0 ]; do + sleep 1 + echo waiting for OVS +done + +set -e + # Configure pipette's OVS switch. -echo "Configuring OVS switch for pipette" +echo "Configuring OVS bridge $BR for pipette" remove_int_ip() { local int="$1" - sudo ip addr flush dev "$int" - sudo ip link set "$int" up + ip addr flush dev "$int" + ip link set "$int" up } -sudo ovs-vsctl --if-exists del-br "$BR" +ovs-vsctl --if-exists del-br "$BR" echo "Configuring bridge" -sudo ovs-vsctl add-br "$BR" +ovs-vsctl add-br "$BR" echo "Adding ports" -sudo ovs-vsctl add-port "$BR" "$COPROINT" -- set Interface "$COPROINT" ofport_request="$COPROPORT" -sudo ovs-vsctl add-port "$BR" "$FAKEINT" -- set Interface "$FAKEINT" ofport_request="$FAKEPORT" type=internal +ovs-vsctl add-port "$BR" "$COPROINT" -- set Interface "$COPROINT" ofport_request="$COPROPORT" +ovs-vsctl add-port "$BR" "$FAKEINT" -- set Interface "$FAKEINT" ofport_request="$FAKEPORT" type=internal echo "Setting controller" -sudo ovs-vsctl set-controller "$BR" tcp:127.0.0.1:"$OF" +ovs-vsctl set-controller "$BR" tcp:127.0.0.1:"$OF" for i in $COPROINT $FAKEINT $BR ; do remove_int_ip "$i" @@ -26,8 +33,13 @@ for ((i=0; i< "${#VLANS[@]}"; i++)) ; do vlan="${VLANS[$i]}" nfvip="${NFVIPS[$i]}" fakeintvlan=${FAKEINT}.${vlan} - sudo ip link add link "$FAKEINT" name "$fakeintvlan" type vlan id "$vlan" - sudo ip link set dev "$fakeintvlan" address "$FAKESERVERMAC" + ip link add link "$FAKEINT" name "$fakeintvlan" type vlan id "$vlan" + ip link set dev "$fakeintvlan" address "$FAKESERVERMAC" remove_int_ip "$fakeintvlan" - sudo ip addr add "$nfvip" dev "$fakeintvlan" + ip addr add "$nfvip" dev "$fakeintvlan" +done + +while [[ "$(ovs-ofctl dump-flows $BR)" == "" ]] ; do + sleep 1 + echo waiting for flows in $BR done diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..895acf5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,53 @@ +version: "3.7" +services: + ovs: + image: iqtlabs/openvswitch:v2.13.1 + volumes: + - /usr/local/var/run/openvswitch:/usr/local/var/run/openvswitch + - ovs-data:/etc/openvswitch + network_mode: host + devices: + - "/dev/net/tun:/dev/net/tun" + cap_add: + - NET_ADMIN + pipette: + image: iqtlabs/pipette:latest + build: + context: . + dockerfile: Dockerfile.pipette + depends_on: + - ovs + ports: + - "${OF}:6653" + environment: + - FAKECLIENTMAC=${FAKECLIENTMAC} + - FAKESERVERMAC=${FAKESERVERMAC} + - NFVIPS=${NFVIPS} + - VLANS=${VLANS} + pipetteconf: + restart: on-failure + image: iqtlabs/pipetteconf:latest + build: + context: . + dockerfile: Dockerfile.pipetteconf + network_mode: host + depends_on: + - ovs + - pipette + volumes: + - /usr/local/var/run/openvswitch:/var/run/openvswitch + environment: + - BR=${BR} + - COPROINT=${COPROINT} + - COPROPORT=${COPROPORT} + - FAKEINT=${FAKEINT} + - FAKEPORT=${FAKEPORT} + - FAKECLIENTMAC=${FAKECLIENTMAC} + - FAKESERVERMAC=${FAKESERVERMAC} + - NFVIPS=${NFVIPS} + - OF=${OF} + - VLANS=${VLANS} + cap_add: + - NET_ADMIN +volumes: + ovs-data: diff --git a/pipetteconf.sh b/pipetteconf.sh index 168a251..ab57a52 100755 --- a/pipetteconf.sh +++ b/pipetteconf.sh @@ -1,41 +1,2 @@ -# pipette configuration - -# NOTE: system management of all pipette's interfaces should be disabled. -# e.g. add denyinterfaces COPROINT, denyinterfaces BR, etc all to /etc/dhcpcd.conf - - -# interface connected to FAUCET coprocessor port. -COPROINT=enx0023565c8859 -# addresses fake services will be run on (will be proxied from real IPs) -# At the moment must be /16, and real network must be /24 (for NAT to work). -# space separated. -NFVIPS="10.10.0.1/16" -# IPv6 is also supported - must be a /64, the real network must be a /96. -# NFVIPS="fc04::1/64" -# FAUCET VLANS where fake services will appear (space separated). -VLANS="2" -# interface that will be created for fake services to run on. -FAKEINT=fake0 -DFILE=Dockerfile - -## -# Optional config -## - -# Reserved MAC addresses for fake services to use to talk to clients. -FAKESERVERMAC=0e:00:00:00:00:66 -FAKECLIENTMAC=0e:00:00:00:00:67 -# OVS bridge name -BR=copro0 -# pipette OF port -OF=6699 -# OF port number for interface facing coprocessor -COPROPORT=1 -# OF port number for fake interface -FAKEPORT=2 -# Flag to record and location to dump pcaps -RECORD=0 -PCAP_LOCATION=./pcaps - -#pipette temporary directory -PIPETTE_TEMP_DIR=/tmp/pipette +# Edit .env to make changes. +source .env diff --git a/runpipette.sh b/runpipette.sh index 67cb207..a4a3603 100755 --- a/runpipette.sh +++ b/runpipette.sh @@ -16,13 +16,11 @@ function show_help() -b, bridge name of ovs bridge to create -p, port pipette port -v, vlans coprocessor vlans, space delimitted - -r, record record traffic captured by pipette should be followed by location then size of file i.e.: -r /pcaps.file.pcap 50 - -n, no-docker run the ryu-manager on the local server instead of in a docker container" + -r, record record traffic captured by pipette should be followed by location then size of file i.e.: -r /pcaps.file.pcap 50" } function check_args() { - NO_DOCKER=0 while [ $# -gt 1 ]; do case $1 in -c|coproint) @@ -74,9 +72,6 @@ function check_args() FILE_SIZE="$2" shift ;; - -n|no-docker) - NO_DOCKER=1 - ;; -h|\?|help) show_help exit @@ -90,13 +85,12 @@ if [ $# -gt 0 ]; then check_args "$@" fi - if [ ! -d "$PIPETTE_TEMP_DIR" ]; then mkdir "$PIPETTE_TEMP_DIR" fi -export BR VLANS COPROINT FAKEINT COPROPORT FAKEPORT FAKESERVERMAC NFVIPS -./configureovs.sh || exit 1 +export BR VLANS COPROINT FAKEINT COPROPORT FAKEPORT FAKESERVERMAC NFVIPS OF +sudo --preserve-env="BR,VLANS,COPROINT,FAKEINT,COPROPORT,FAKEPORT,FAKESERVERMAC,NFVIPS,OF" ./configureovs.sh || exit 1 if [ $RECORD -ne 0 ]; then echo "Starting tcpdump on interface $COPROINT" @@ -104,12 +98,5 @@ if [ $RECORD -ne 0 ]; then echo $! >> "$PIPETTE_TEMP_DIR/tcpdump" fi -if [[ NO_DOCKER -ne 0 ]]; then - ryu-manager --verbose --ofp-tcp-listen-port "$OF" pipette.py & - echo $! >> "$PIPETTE_TEMP_DIR/ryu" -else - docker build -f $DFILE . -t iqtlabs/pipette - docker_id=$(docker run -d -e NFVIPS="$NFVIPS" -e FAKESERVERMAC="$FAKESERVERMAC" -e FAKECLIENTMAC="$FAKECLIENTMAC" -e VLANS="$VLANS" \ - -p 127.0.0.1:$OF:6653 -ti iqtlabs/pipette) - echo "$docker_id" >> "$PIPETTE_TEMP_DIR/ryu.docker" -fi +ryu-manager --verbose --ofp-tcp-listen-port "$OF" pipette.py & +echo $! >> "$PIPETTE_TEMP_DIR/ryu" diff --git a/shutdownpipette.sh b/shutdownpipette.sh index 13b7146..0625125 100755 --- a/shutdownpipette.sh +++ b/shutdownpipette.sh @@ -43,14 +43,6 @@ if [ $# -gt 0 ]; then check_args "$@" fi - -if [ -f "$PIPETTE_TEMP_DIR/ryu.docker" ]; then - docker_id=$(cat "$PIPETTE_TEMP_DIR/ryu.docker") - echo "stopping container with id $docker_id" - sudo docker stop "$docker_id" - sudo docker rm "$docker_id" -fi - if [ -f "$PIPETTE_TEMP_DIR/ryu" ]; then ryu_pid=$(cat "$PIPETTE_TEMP_DIR/ryu") echo "killing process with pid $ryu_pid" From aa81258b183930874f320bd2e0737338b10e23b6 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Tue, 19 Jan 2021 19:52:13 +1300 Subject: [PATCH 2/2] conf tag. --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 54ff6b2..0a30429 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -45,5 +45,5 @@ jobs: docker buildx build \ --platform linux/amd64,linux/arm/v7,linux/arm64 \ --push \ - -t iqtlabs/pipette:${{ steps.change_version.outputs.VERSION }} -f Dockerfile.pipetteconf . + -t iqtlabs/pipetteconf:${{ steps.change_version.outputs.VERSION }} -f Dockerfile.pipetteconf . if: github.repository == 'iqtlabs/pipette' && github.event_name == 'push'