From 6ae3e27ccb6ca7aef473fd017a27e097e7886220 Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Mon, 18 Nov 2024 16:42:52 +0100 Subject: [PATCH 1/2] filter regexes --- res/flow-capture.yml | 2 +- res/packet-capture.yml | 2 +- scripts/functions.sh | 464 +++++++++++++++++++++++------------------ 3 files changed, 260 insertions(+), 208 deletions(-) diff --git a/res/flow-capture.yml b/res/flow-capture.yml index e264507a..c7432002 100644 --- a/res/flow-capture.yml +++ b/res/flow-capture.yml @@ -146,7 +146,7 @@ spec: "name":"send", "follows":"enrich" } - ], + ] } volumeMounts: - name: bpf-kernel-debug diff --git a/res/packet-capture.yml b/res/packet-capture.yml index bdd589f7..38af3bc3 100644 --- a/res/packet-capture.yml +++ b/res/packet-capture.yml @@ -130,7 +130,7 @@ spec: "name":"send", "follows":"enrich" } - ], + ] } volumeMounts: - name: bpf-kernel-debug diff --git a/scripts/functions.sh b/scripts/functions.sh index 4ed51140..b4261a03 100755 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -13,8 +13,8 @@ if [ -z "${copy+x}" ]; then copy="prompt"; fi # get either oc (favorite) or kubectl paths # this is used only when calling commands directly # else it will be overridden by inject.sh -K8S_CLI_BIN_PATH=$( which oc 2>/dev/null || which kubectl 2>/dev/null ) -K8S_CLI_BIN=$( basename "${K8S_CLI_BIN_PATH}" ) +K8S_CLI_BIN_PATH=$(which oc 2>/dev/null || which kubectl 2>/dev/null) +K8S_CLI_BIN=$(basename "${K8S_CLI_BIN_PATH}") # eBPF agent image to use agentImg="quay.io/netobserv/netobserv-ebpf-agent:main" @@ -64,20 +64,21 @@ function loadYAMLs() { } function clusterIsReady() { - # use oc whoami as connectivity check by default and fallback to kubectl get all if needed - K8S_CLI_CONNECTIVITY="${K8S_CLI_BIN} whoami" - if [ "${K8S_CLI_BIN}" = "kubectl" ]; then - K8S_CLI_CONNECTIVITY="${K8S_CLI_BIN} get all" - fi - if ${K8S_CLI_CONNECTIVITY} 2>&1 || ${K8S_CLI_BIN} cluster-info | grep -q "Kubernetes control plane"; then - return 0 - else - return 1 - fi + # use oc whoami as connectivity check by default and fallback to kubectl get all if needed + K8S_CLI_CONNECTIVITY="${K8S_CLI_BIN} whoami" + if [ "${K8S_CLI_BIN}" = "kubectl" ]; then + K8S_CLI_CONNECTIVITY="${K8S_CLI_BIN} get all" + fi + if ${K8S_CLI_CONNECTIVITY} 2>&1 || ${K8S_CLI_BIN} cluster-info | grep -q "Kubernetes control plane"; then + return 0 + else + return 1 + fi } FLOWS_MANIFEST_FILE="flow-capture.yml" PACKETS_MANIFEST_FILE="packet-capture.yml" +CONFIG_JSON_TEMP="config.json" MANIFEST_OUTPUT_PATH="tmp" function setup { @@ -111,20 +112,20 @@ function setup { shift echo "creating flow-capture agents:" if [[ ! -d ${MANIFEST_OUTPUT_PATH} ]]; then - mkdir -p ${MANIFEST_OUTPUT_PATH} > /dev/null + mkdir -p ${MANIFEST_OUTPUT_PATH} >/dev/null fi manifest="${MANIFEST_OUTPUT_PATH}/${FLOWS_MANIFEST_FILE}" - echo "${flowAgentYAML}" > ${manifest} + echo "${flowAgentYAML}" >${manifest} options="$*" check_args_and_apply "$options" "$manifest" "flows" elif [ "$1" = "packets" ]; then shift echo "creating packet-capture agents" if [[ ! -d ${MANIFEST_OUTPUT_PATH} ]]; then - mkdir -p ${MANIFEST_OUTPUT_PATH} > /dev/null + mkdir -p ${MANIFEST_OUTPUT_PATH} >/dev/null fi manifest="${MANIFEST_OUTPUT_PATH}/${PACKETS_MANIFEST_FILE}" - echo "${packetAgentYAML}" > ${manifest} + echo "${packetAgentYAML}" >${manifest} options="$*" check_args_and_apply "$options" "$manifest" "packets" fi @@ -145,12 +146,18 @@ function cleanup { copyOutput elif [ "$copy" = "prompt" ]; then while true; do - read -rp "Copy the capture output locally ?" yn - case $yn in - [Yy]* ) copyOutput; break;; - [Nn]* ) echo "copy skipped"; break;; - * ) echo "Please answer yes or no.";; - esac + read -rp "Copy the capture output locally ?" yn + case $yn in + [Yy]*) + copyOutput + break + ;; + [Nn]*) + echo "copy skipped" + break + ;; + *) echo "Please answer yes or no." ;; + esac done fi @@ -188,6 +195,7 @@ function common_usage { echo " --icmp_code: filter ICMP code (default: n/a)" echo " --peer_ip: filter peer IP (default: n/a)" echo " --drops: filter flows with only dropped packets (default: false)" + echo " --regexes: filter flows using regex (default: n/a)" } function flows_usage { @@ -285,6 +293,41 @@ function edit_manifest() { "filter_pkt_drops") yq e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"FILTER_DROPS\").value|=\"$2\"" "$3" ;; + "filter_regexes") + # get current config and save it to temp file + jsonContent=$(yq e '.spec.template.spec.containers[0].env[] | select(.name=="FLP_CONFIG").value' "$3") + json="${MANIFEST_OUTPUT_PATH}/${CONFIG_JSON_TEMP}" + echo "$jsonContent" >${json} + + # remove send step + yq e -oj --inplace "del(.pipeline[] | select(.name==\"send\"))" "$json" + + # define rules from arg + rules=() + for regex in $(echo $2 | tr "," "\n"); do + keyValue=(${regex//"~"/ }) + key=${keyValue[0]} + value=${keyValue[1]} + echo "key: $key value: $value" + rules+=("{\"type\":\"keep_entry_if_regex_match\",\"keepEntry\":{\"input\":\"$key\",\"value\":\"$value\"}}") + done + echo "rules: ${rules[@]}" + rules=$(echo "${rules[@]}" | sed "s/ /,/g") + + # add filter param & pipeline + yq e -oj --inplace ".parameters += {\"name\":\"filter\",\"transform\":{\"type\":\"filter\",\"filter\":{\"rules\":[{\"type\":\"keep_entry_all_satisfied\",\"keepEntryAllSatisfied\":[$rules]}]}}}" "$json" + yq e -oj --inplace ".pipeline += {\"name\":\"filter\",\"follows\":\"enrich\"}" "$json" + + # add send step back + yq e -oj --inplace ".pipeline += {\"name\":\"send\",\"follows\":\"filter\"}" "$json" + + # get json as string with escaped quotes + jsonContent=$(cat $json) + jsonContent=${jsonContent//\"/\\\"} + + # update FLP_CONFIG env + yq e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"FLP_CONFIG\").value|=\"$jsonContent\"" "$3" + ;; "log_level") yq e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"LOG_LEVEL\").value|=\"$2\"" "$3" ;; @@ -301,188 +344,197 @@ function edit_manifest() { #$2: manifest #$3: either flows or packets function check_args_and_apply() { - # Iterate through the command-line arguments - for option in $1; do - key="${option%%=*}" - value="${option#*=}" - case "$key" in - --copy) # Copy or skip without prompt - if [[ "$value" == "true" || "$value" == "false" || "$value" == "prompt" ]]; then - copy="$value" - else - echo "invalid value for --copy" - fi - ;; - --interfaces) # Interfaces - edit_manifest "interfaces" "$value" "$2" - ;; - --enable_pktdrop) # Enable packet drop - if [[ "$3" == "flows" ]]; then - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "pktdrop_enable" "$value" "$2" - else - echo "invalid value for --enable_pktdrop" - fi - else - echo "--enable_pktdrop is invalid option for packets" - exit 1 - fi - ;; - --enable_dns) # Enable DNS - if [[ "$3" == "flows" ]]; then - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "dns_enable" "$value" "$2" - else - echo "invalid value for --enable_dns" - fi - else - echo "--enable_dns is invalid option for packets" - exit 1 - fi - ;; - --enable_rtt) # Enable RTT - if [[ "$3" == "flows" ]]; then - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "rtt_enable" "$value" "$2" - else - echo "invalid value for --enable_rtt" - fi - else - echo "--enable_rtt is invalid option for packets" - exit 1 - fi - ;; - --enable_network_events) # Enable Network events monitoring - if [[ "$3" == "flows" ]]; then - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "network_events_enable" "$value" "$2" - else - echo "invalid value for --enable_network_events" - fi - else - echo "--enable_network_events is invalid option for packets" - exit 1 - fi - ;; - --enable_filter) # Enable flow filter - if [[ "$3" == "flows" ]]; then - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "filter_enable" "$value" "$2" - else - echo "invalid value for --enable_filter" - fi - else - echo "--enable_filter is invalid option for packets" - exit 1 - fi - ;; - --direction) # Configure filter direction - if [[ "$value" == "Ingress" || "$value" == "Egress" ]]; then - edit_manifest "filter_direction" "$value" "$2" - else - echo "invalid value for --direction" - fi - ;; - --cidr) # Configure flow CIDR - edit_manifest "filter_cidr" "$value" "$2" - ;; - --protocol) # Configure filter protocol - if [[ "$value" == "TCP" || "$value" == "UDP" || "$value" == "SCTP" || "$value" == "ICMP" || "$value" == "ICMPv6" ]]; then - edit_manifest "filter_protocol" "$value" "$2" - else - echo "invalid value for --protocol" - fi - ;; - --sport) # Configure filter source port - edit_manifest "filter_sport" "$value" "$2" - ;; - --dport) # Configure filter destination port - edit_manifest "filter_dport" "$value" "$2" - ;; - --port) # Configure filter port - edit_manifest "filter_port" "$value" "$2" - ;; - --sport_range) # Configure filter source port range - edit_manifest "filter_sport_range" "$value" "$2" - ;; - --dport_range) # Configure filter destination port range - edit_manifest "filter_dport_range" "$value" "$2" - ;; - --port_range) # Configure filter port range - edit_manifest "filter_port_range" "$value" "$2" - ;; - --sports) # Configure filter source two ports using "," - edit_manifest "filter_sports" "$value" "$2" - ;; - --dports) # Configure filter destination two ports using "," - edit_manifest "filter_dports" "$value" "$2" - ;; - --ports) # Configure filter on two ports usig "," can either be srcport or dstport - edit_manifest "filter_ports" "$value" "$2" - ;; - --tcp_flags) # Configure filter TCP flags - if [[ "$value" == "SYN" || "$value" == "SYN-ACK" || "$value" == "ACK" || "$value" == "FIN" || "$value" == "RST" || "$value" == "FIN-ACK" || "$value" == "RST-ACK" || "$value" == "PSH" || "$value" == "URG" || "$value" == "ECE" || "$value" == "CWR" ]]; then - edit_manifest "filter_tcp_flags" "$value" "$2" - else - echo "invalid value for --tcp_flags" - fi - ;; - --drops) # Filter packet drops - if [[ "$value" == "true" || "$value" == "false" ]]; then - edit_manifest "filter_pkt_drops" "$value" "$2" - else - echo "invalid value for --drops" - fi - ;; - --icmp_type) # ICMP type - edit_manifest "filter_icmp_type" "$value" "$2" - ;; - --icmp_code) # ICMP code - edit_manifest "filter_icmp_code" "$value" "$2" - ;; - --peer_ip) # Peer IP - edit_manifest "filter_peer_ip" "$value" "$2" - ;; - --action) # Filter action - if [[ "$value" == "Accept" || "$value" == "Reject" ]]; then - edit_manifest "filter_action" "$value" "$2" - else - echo "invalid value for --action" - fi - ;; - --log-level) # Log level - if [[ "$value" == "trace" || "$value" == "debug" || "$value" == "info" || "$value" == "warn" || "$value" == "error" || "$value" == "fatal" || "$value" == "panic" ]]; then - edit_manifest "log_level" "$value" "$2" - logLevel=$value - filter=${filter/$key=$logLevel/} - else - echo "invalid value for --action" - fi - ;; - --max-time) # Max time - maxTime=$value - filter=${filter/$key=$maxTime/} - ;; - --max-bytes) # Max bytes - maxBytes=$value - filter=${filter/$key=$maxBytes/} - ;; - --node-selector) # Node selector - if [[ $value == *":"* ]]; then - edit_manifest "node_selector" "$value" "$2" - else - echo "invalid value for --node-selector. Use --node-selector=key:val instead." - exit 1 - fi - ;; - *) # Invalid option - echo "Invalid option: $key" >&2 - exit 1 - ;; - esac - done + # Iterate through the command-line arguments + for option in $1; do + key="${option%%=*}" + value="${option#*=}" + case "$key" in + --copy) # Copy or skip without prompt + if [[ "$value" == "true" || "$value" == "false" || "$value" == "prompt" ]]; then + copy="$value" + else + echo "invalid value for --copy" + fi + ;; + --interfaces) # Interfaces + edit_manifest "interfaces" "$value" "$2" + ;; + --enable_pktdrop) # Enable packet drop + if [[ "$3" == "flows" ]]; then + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "pktdrop_enable" "$value" "$2" + else + echo "invalid value for --enable_pktdrop" + fi + else + echo "--enable_pktdrop is invalid option for packets" + exit 1 + fi + ;; + --enable_dns) # Enable DNS + if [[ "$3" == "flows" ]]; then + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "dns_enable" "$value" "$2" + else + echo "invalid value for --enable_dns" + fi + else + echo "--enable_dns is invalid option for packets" + exit 1 + fi + ;; + --enable_rtt) # Enable RTT + if [[ "$3" == "flows" ]]; then + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "rtt_enable" "$value" "$2" + else + echo "invalid value for --enable_rtt" + fi + else + echo "--enable_rtt is invalid option for packets" + exit 1 + fi + ;; + --enable_network_events) # Enable Network events monitoring + if [[ "$3" == "flows" ]]; then + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "network_events_enable" "$value" "$2" + else + echo "invalid value for --enable_network_events" + fi + else + echo "--enable_network_events is invalid option for packets" + exit 1 + fi + ;; + --enable_filter) # Enable flow filter + if [[ "$3" == "flows" ]]; then + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "filter_enable" "$value" "$2" + else + echo "invalid value for --enable_filter" + fi + else + echo "--enable_filter is invalid option for packets" + exit 1 + fi + ;; + --direction) # Configure filter direction + if [[ "$value" == "Ingress" || "$value" == "Egress" ]]; then + edit_manifest "filter_direction" "$value" "$2" + else + echo "invalid value for --direction" + fi + ;; + --cidr) # Configure flow CIDR + edit_manifest "filter_cidr" "$value" "$2" + ;; + --protocol) # Configure filter protocol + if [[ "$value" == "TCP" || "$value" == "UDP" || "$value" == "SCTP" || "$value" == "ICMP" || "$value" == "ICMPv6" ]]; then + edit_manifest "filter_protocol" "$value" "$2" + else + echo "invalid value for --protocol" + fi + ;; + --sport) # Configure filter source port + edit_manifest "filter_sport" "$value" "$2" + ;; + --dport) # Configure filter destination port + edit_manifest "filter_dport" "$value" "$2" + ;; + --port) # Configure filter port + edit_manifest "filter_port" "$value" "$2" + ;; + --sport_range) # Configure filter source port range + edit_manifest "filter_sport_range" "$value" "$2" + ;; + --dport_range) # Configure filter destination port range + edit_manifest "filter_dport_range" "$value" "$2" + ;; + --port_range) # Configure filter port range + edit_manifest "filter_port_range" "$value" "$2" + ;; + --sports) # Configure filter source two ports using "," + edit_manifest "filter_sports" "$value" "$2" + ;; + --dports) # Configure filter destination two ports using "," + edit_manifest "filter_dports" "$value" "$2" + ;; + --ports) # Configure filter on two ports usig "," can either be srcport or dstport + edit_manifest "filter_ports" "$value" "$2" + ;; + --tcp_flags) # Configure filter TCP flags + if [[ "$value" == "SYN" || "$value" == "SYN-ACK" || "$value" == "ACK" || "$value" == "FIN" || "$value" == "RST" || "$value" == "FIN-ACK" || "$value" == "RST-ACK" || "$value" == "PSH" || "$value" == "URG" || "$value" == "ECE" || "$value" == "CWR" ]]; then + edit_manifest "filter_tcp_flags" "$value" "$2" + else + echo "invalid value for --tcp_flags" + fi + ;; + --drops) # Filter packet drops + if [[ "$value" == "true" || "$value" == "false" ]]; then + edit_manifest "filter_pkt_drops" "$value" "$2" + else + echo "invalid value for --drops" + fi + ;; + --regexes) # Filter using regexes + valueCount=$(grep -o "~" <<<"$value" | wc -l) + splitterCount=$(grep -o "," <<<"$value" | wc -l) + if [[ valueCount > 0 && $((valueCount)) == $((splitterCount + 1)) ]]; then + edit_manifest "filter_regexes" "$value" "$2" + else + echo "invalid value for --regexes" + fi + ;; + --icmp_type) # ICMP type + edit_manifest "filter_icmp_type" "$value" "$2" + ;; + --icmp_code) # ICMP code + edit_manifest "filter_icmp_code" "$value" "$2" + ;; + --peer_ip) # Peer IP + edit_manifest "filter_peer_ip" "$value" "$2" + ;; + --action) # Filter action + if [[ "$value" == "Accept" || "$value" == "Reject" ]]; then + edit_manifest "filter_action" "$value" "$2" + else + echo "invalid value for --action" + fi + ;; + --log-level) # Log level + if [[ "$value" == "trace" || "$value" == "debug" || "$value" == "info" || "$value" == "warn" || "$value" == "error" || "$value" == "fatal" || "$value" == "panic" ]]; then + edit_manifest "log_level" "$value" "$2" + logLevel=$value + filter=${filter/$key=$logLevel/} + else + echo "invalid value for --action" + fi + ;; + --max-time) # Max time + maxTime=$value + filter=${filter/$key=$maxTime/} + ;; + --max-bytes) # Max bytes + maxBytes=$value + filter=${filter/$key=$maxBytes/} + ;; + --node-selector) # Node selector + if [[ $value == *":"* ]]; then + edit_manifest "node_selector" "$value" "$2" + else + echo "invalid value for --node-selector. Use --node-selector=key:val instead." + exit 1 + fi + ;; + *) # Invalid option + echo "Invalid option: $key" >&2 + exit 1 + ;; + esac + done - ${K8S_CLI_BIN} apply -f "$2" - ${K8S_CLI_BIN} rollout status daemonset netobserv-cli -n netobserv-cli --timeout 60s - rm -rf ${MANIFEST_OUTPUT_PATH} -} \ No newline at end of file + ${K8S_CLI_BIN} apply -f "$2" + ${K8S_CLI_BIN} rollout status daemonset netobserv-cli -n netobserv-cli --timeout 60s + rm -rf ${MANIFEST_OUTPUT_PATH} +} From c53740bad3735387417b844510640cf974b59e48 Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Tue, 19 Nov 2024 10:55:37 +0100 Subject: [PATCH 2/2] lint --- scripts/functions.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/functions.sh b/scripts/functions.sh index b4261a03..fe8c7dda 100755 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -303,19 +303,22 @@ function edit_manifest() { yq e -oj --inplace "del(.pipeline[] | select(.name==\"send\"))" "$json" # define rules from arg + IFS=',' read -ra regexes <<<"$2" rules=() - for regex in $(echo $2 | tr "," "\n"); do - keyValue=(${regex//"~"/ }) + for regex in "${regexes[@]}"; do + IFS='~' read -ra keyValue <<<"$regex" key=${keyValue[0]} value=${keyValue[1]} echo "key: $key value: $value" rules+=("{\"type\":\"keep_entry_if_regex_match\",\"keepEntry\":{\"input\":\"$key\",\"value\":\"$value\"}}") done - echo "rules: ${rules[@]}" - rules=$(echo "${rules[@]}" | sed "s/ /,/g") + rulesStr=$( + IFS=, + echo "${rules[*]}" + ) # add filter param & pipeline - yq e -oj --inplace ".parameters += {\"name\":\"filter\",\"transform\":{\"type\":\"filter\",\"filter\":{\"rules\":[{\"type\":\"keep_entry_all_satisfied\",\"keepEntryAllSatisfied\":[$rules]}]}}}" "$json" + yq e -oj --inplace ".parameters += {\"name\":\"filter\",\"transform\":{\"type\":\"filter\",\"filter\":{\"rules\":[{\"type\":\"keep_entry_all_satisfied\",\"keepEntryAllSatisfied\":[$rulesStr]}]}}}" "$json" yq e -oj --inplace ".pipeline += {\"name\":\"filter\",\"follows\":\"enrich\"}" "$json" # add send step back @@ -480,7 +483,7 @@ function check_args_and_apply() { --regexes) # Filter using regexes valueCount=$(grep -o "~" <<<"$value" | wc -l) splitterCount=$(grep -o "," <<<"$value" | wc -l) - if [[ valueCount > 0 && $((valueCount)) == $((splitterCount + 1)) ]]; then + if [[ ${valueCount} -gt 0 && $((valueCount)) == $((splitterCount + 1)) ]]; then edit_manifest "filter_regexes" "$value" "$2" else echo "invalid value for --regexes"