COS-2822: port gcp-routes to nftables

overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Update iptables rules based on google cloud load balancer VIPS
+# Update nftables rules based on google cloud load balancer VIPS
 #
 # This is needed because the GCP L3 load balancer doesn't actually do DNAT;
 # the destination IP address is still the VIP. Normally, there is an agent that
@@ -14,8 +14,6 @@
 #
 # Additionally, clients can write a file to /run/gcp-routes/$IP.down to force
 # a VIP as down. This is useful for graceful shutdown / upgrade.
-#
-# ~cdc~
 
 set -e
 
@@ -27,63 +25,41 @@ curler() {
    curl --silent -L -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/${1}"
 }
 
-CHAIN_NAME="gcp-vips"
+TABLE_NAME="gcp-vips"
+EXTERNAL_VIPS_CHAIN="external-vips"
 RUN_DIR="/run/gcp-routes"
 
-# Create a chan if it doesn't exist
-ensure_chain() {
-    local table="${1}"
-    local chain="${2}"
-
-    if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then
-        iptables -w -t "${table}" -N "${chain}";
-    fi;
-}
-
-ensure_rule() {
-    local table="${1}"
-    local chain="${2}"
-    shift 2
-
-    if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then
-        iptables -w -t "${table}" -A "${chain}" "$@"
-    fi
-}
-
-# set the chain, ensure entry rules, ensure ESTABLISHED rule
+# Set up base table and rules
 initialize() {
-    ensure_chain nat "${CHAIN_NAME}"
-    ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
-
-    # Need this so that existing flows (with an entry in conntrack) continue to be
-    # balanced, even if the DNAT entry is removed
-    ensure_rule filter INPUT -m comment --comment 'gcp LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT
+    nft -f - <<EOF
+        add table ip ${TABLE_NAME} { comment "apiserver loadbalancer routing helper"; }
+        add chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} { type nat hook prerouting priority dstnat; comment "gcp LB vip DNAT for external clients"; }
+EOF
 
     mkdir -p "${RUN_DIR}"
 }
 
-remove_stale() {
-    ## find extra iptables rules
-    for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
-        if [[ -z "${vips[${ipt_vip}]}" ]]; then
-            echo removing stale vip "${ipt_vip}"
-            iptables -w -t nat -D "${CHAIN_NAME}" --dst "${ipt_vip}" -j REDIRECT
-        fi
+sync_rules() {
+    # Construct the VIP lists. (The nftables syntax allows a trailing comma.)
+    external_vips=""
+    for vip in "${!vips[@]}"; do
+        external_vips="${vip}, ${external_vips}"
     done
-}
 
-add_rules() {
-    for vip in "${vips[@]}"; do
-        echo "ensuring rule for ${vip}"
-        ensure_rule nat "${CHAIN_NAME}" --dst "${vip}" -j REDIRECT
-    done
+    echo "synchronizing VIPs to (${external_vips})"
+    {
+        echo "flush chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN}"
+        if [[ -n "${external_vips}" ]]; then
+            echo "add rule ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} ip daddr { ${external_vips} } redirect"
+        fi
+    } | nft -f -
 }
 
 clear_rules() {
-    iptables -t nat -F "${CHAIN_NAME}" || true
+    nft delete table ip "${TABLE_NAME}" || true
 }
 
-# out paramater: vips
+# out parameter: vips
 list_lb_ips() {
     for k in "${!vips[@]}"; do
         unset vips["${k}"]
@@ -128,8 +104,7 @@ case "$1" in
     initialize
     while :; do
       list_lb_ips
-      remove_stale
-      add_rules
+      sync_rules
       echo "done applying vip rules"
       sleep_or_watch
     done