From e1d85f26dd8d6dca719a1ae36209b0bc5efd8461 Mon Sep 17 00:00:00 2001
From: Dan Winship <danwinship@redhat.com>
Date: Thu, 8 Aug 2024 10:04:51 -0400
Subject: [PATCH] Port gcp-routes to nftables

---
 overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh | 67 +++++++------------
 1 file changed, 23 insertions(+), 44 deletions(-)

diff --git a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
index 64a592c2..39fdba5c 100755
--- a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
+++ b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Update iptables rules based on google cloud load balancer VIPS
+# Update nftables rules based on google cloud load balancer VIPS
 #
 # This is needed because the GCP L3 load balancer doesn't actually do DNAT;
 # the destination IP address is still the VIP. Normally, there is an agent that
@@ -14,8 +14,6 @@
 #
 # Additionally, clients can write a file to /run/gcp-routes/$IP.down to force
 # a VIP as down. This is useful for graceful shutdown / upgrade.
-#
-# ~cdc~
 
 set -e
 
@@ -27,59 +25,41 @@ curler() {
    curl --silent -L -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/${1}"
 }
 
-CHAIN_NAME="gcp-vips"
+TABLE_NAME="gcp-vips"
+EXTERNAL_VIPS_CHAIN="external-vips"
 RUN_DIR="/run/gcp-routes"
 
-# Create a chan if it doesn't exist
-ensure_chain() {
-    local table="${1}"
-    local chain="${2}"
-
-    if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then
-        iptables -w -t "${table}" -N "${chain}";
-    fi;
-}
-
-ensure_rule() {
-    local table="${1}"
-    local chain="${2}"
-    shift 2
-
-    if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then
-        iptables -w -t "${table}" -A "${chain}" "$@"
-    fi
-}
-
-# set the chain, ensure entry rules, ensure ESTABLISHED rule
+# Set up base table and rules
 initialize() {
-    ensure_chain nat "${CHAIN_NAME}"
-    ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
+    nft -f - <<EOF
+        add table ip ${TABLE_NAME} { comment "apiserver loadbalancer routing helper"; }
+        add chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} { type nat hook prerouting priority dstnat; comment "gcp LB vip DNAT for external clients"; }
+EOF
 
     mkdir -p "${RUN_DIR}"
 }
 
-remove_stale() {
-    ## find extra iptables rules
-    for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
-        if [[ -z "${vips[${ipt_vip}]}" ]]; then
-            echo removing stale vip "${ipt_vip}"
-            iptables -w -t nat -D "${CHAIN_NAME}" --dst "${ipt_vip}" -j REDIRECT
-        fi
+sync_rules() {
+    # Construct the VIP lists. (The nftables syntax allows a trailing comma.)
+    external_vips=""
+    for vip in "${!vips[@]}"; do
+        external_vips="${vip}, ${external_vips}"
     done
-}
 
-add_rules() {
-    for vip in "${vips[@]}"; do
-        echo "ensuring rule for ${vip}"
-        ensure_rule nat "${CHAIN_NAME}" --dst "${vip}" -j REDIRECT
-    done
+    echo "synchronizing VIPs to (${external_vips})"
+    {
+        echo "flush chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN}"
+        if [[ -n "${external_vips}" ]]; then
+            echo "add rule ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} ip daddr { ${external_vips} } redirect"
+        fi
+    } | nft -f -
 }
 
 clear_rules() {
-    iptables -t nat -F "${CHAIN_NAME}" || true
+    nft delete table ip "${TABLE_NAME}" || true
 }
 
-# out paramater: vips
+# out parameter: vips
 list_lb_ips() {
     for k in "${!vips[@]}"; do
         unset vips["${k}"]
@@ -124,8 +104,7 @@ case "$1" in
     initialize
     while :; do
       list_lb_ips
-      remove_stale
-      add_rules
+      sync_rules
       echo "done applying vip rules"
       sleep_or_watch
     done