From a5085f526ec2306ce654c699ff29e193e714e8b1 Mon Sep 17 00:00:00 2001
From: Dan Winship <danwinship@redhat.com>
Date: Thu, 8 Aug 2024 09:52:10 -0400
Subject: [PATCH 1/2] Remove unnecessary gcp-routes rule

gcp-routes had a rule "so that existing flows (with an entry in
conntrack) continue to be balanced, even if the DNAT entry is
removed". The only way this iptables rule would actually be needed is
if (a) your masters have an iptables-based firewall (which they
shouldn't, on OCP), and (b) the firewall is so aggressive that it even
drops packets from established connections (which no firewall should
do anyway).

At any rate, even if the rule *was* necessary in some clusters, it
won't work in future nftables-only versions of RHCOS anyway, because
nftables doesn't let "accept" rules in one table override
"drop"/"reject" rules in another table; if your firewall is broken and
dropping packets that it shouldn't, you have to actually fix your
firewall rules, not hack around them somewhere else.
---
 overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
index 5876fe56..64a592c2 100755
--- a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
+++ b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
@@ -55,10 +55,6 @@ initialize() {
     ensure_chain nat "${CHAIN_NAME}"
     ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
 
-    # Need this so that existing flows (with an entry in conntrack) continue to be
-    # balanced, even if the DNAT entry is removed
-    ensure_rule filter INPUT -m comment --comment 'gcp LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT
-
     mkdir -p "${RUN_DIR}"
 }
 

From 29c6c1cbb2a35fd4e4e4b41f1d29ff177ac2f5fb Mon Sep 17 00:00:00 2001
From: Dan Winship <danwinship@redhat.com>
Date: Thu, 8 Aug 2024 10:04:51 -0400
Subject: [PATCH 2/2] Port gcp-routes to nftables

IPTables is going away in RHEL 10 so everything needs to be done with nftables.

https://issues.redhat.com/browse/OCPSTRAT-940
---
 overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh | 67 +++++++------------
 1 file changed, 23 insertions(+), 44 deletions(-)

diff --git a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
index 64a592c2..39fdba5c 100755
--- a/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
+++ b/overlay.d/06gcp-routes/usr/sbin/gcp-routes.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Update iptables rules based on google cloud load balancer VIPS
+# Update nftables rules based on google cloud load balancer VIPS
 #
 # This is needed because the GCP L3 load balancer doesn't actually do DNAT;
 # the destination IP address is still the VIP. Normally, there is an agent that
@@ -14,8 +14,6 @@
 #
 # Additionally, clients can write a file to /run/gcp-routes/$IP.down to force
 # a VIP as down. This is useful for graceful shutdown / upgrade.
-#
-# ~cdc~
 
 set -e
 
@@ -27,59 +25,41 @@ curler() {
    curl --silent -L -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/${1}"
 }
 
-CHAIN_NAME="gcp-vips"
+TABLE_NAME="gcp-vips"
+EXTERNAL_VIPS_CHAIN="external-vips"
 RUN_DIR="/run/gcp-routes"
 
-# Create a chan if it doesn't exist
-ensure_chain() {
-    local table="${1}"
-    local chain="${2}"
-
-    if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then
-        iptables -w -t "${table}" -N "${chain}";
-    fi;
-}
-
-ensure_rule() {
-    local table="${1}"
-    local chain="${2}"
-    shift 2
-
-    if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then
-        iptables -w -t "${table}" -A "${chain}" "$@"
-    fi
-}
-
-# set the chain, ensure entry rules, ensure ESTABLISHED rule
+# Set up base table and rules
 initialize() {
-    ensure_chain nat "${CHAIN_NAME}"
-    ensure_rule nat PREROUTING -m comment --comment 'gcp LB vip DNAT' -j ${CHAIN_NAME}
+    nft -f - <<EOF
+        add table ip ${TABLE_NAME} { comment "apiserver loadbalancer routing helper"; }
+        add chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} { type nat hook prerouting priority dstnat; comment "gcp LB vip DNAT for external clients"; }
+EOF
 
     mkdir -p "${RUN_DIR}"
 }
 
-remove_stale() {
-    ## find extra iptables rules
-    for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}" | awk '$4{print $4}' | awk -F/ '{print $1}'); do
-        if [[ -z "${vips[${ipt_vip}]}" ]]; then
-            echo removing stale vip "${ipt_vip}"
-            iptables -w -t nat -D "${CHAIN_NAME}" --dst "${ipt_vip}" -j REDIRECT
-        fi
+sync_rules() {
+    # Construct the VIP lists. (The nftables syntax allows a trailing comma.)
+    external_vips=""
+    for vip in "${!vips[@]}"; do
+        external_vips="${vip}, ${external_vips}"
     done
-}
 
-add_rules() {
-    for vip in "${vips[@]}"; do
-        echo "ensuring rule for ${vip}"
-        ensure_rule nat "${CHAIN_NAME}" --dst "${vip}" -j REDIRECT
-    done
+    echo "synchronizing VIPs to (${external_vips})"
+    {
+        echo "flush chain ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN}"
+        if [[ -n "${external_vips}" ]]; then
+            echo "add rule ip ${TABLE_NAME} ${EXTERNAL_VIPS_CHAIN} ip daddr { ${external_vips} } redirect"
+        fi
+    } | nft -f -
 }
 
 clear_rules() {
-    iptables -t nat -F "${CHAIN_NAME}" || true
+    nft delete table ip "${TABLE_NAME}" || true
 }
 
-# out paramater: vips
+# out parameter: vips
 list_lb_ips() {
     for k in "${!vips[@]}"; do
         unset vips["${k}"]
@@ -124,8 +104,7 @@ case "$1" in
     initialize
     while :; do
       list_lb_ips
-      remove_stale
-      add_rules
+      sync_rules
       echo "done applying vip rules"
       sleep_or_watch
     done