Override MAC address in Tx eBPF program for kubemark (#661)

* Override MAC address in Tx eBPF program for kubemark * Set MAC override only when valid MAC is found
CentaurusInfra · Sep 9, 2022 · d2ea874 · d2ea874
1 parent 00b2579
commit d2ea874
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 1 deletion.
diff --git a/etc/deploy/deploy.mizar.dev.yaml b/etc/deploy/deploy.mizar.dev.yaml
@@ -523,6 +523,8 @@ spec:
       - name: mizar-daemon
         image: mizarnet/dropletd:dev
         env:
+        - name: FEATUREGATE_KUBEMARK_NETWORK_PERF
+          value: 'false'
         - name: FEATUREGATE_BWQOS
           value: 'false'
         securityContext:
@@ -567,6 +569,8 @@ spec:
       - name: mizar-operator
         image: mizarnet/endpointopr:dev
         env:
+        - name: FEATUREGATE_KUBEMARK_NETWORK_PERF
+          value: 'false'
         - name: FEATUREGATE_BWQOS
           value: 'false'
         securityContext:

diff --git a/mizar/daemon/interface_service.py b/mizar/daemon/interface_service.py
@@ -576,6 +576,23 @@ def update_agent_metadata(self, interface):
         bouncers = []
         for bouncer in interface.bouncers:
             bouncers.append(bouncer.ip_address)
+        dst_mac = ""
+        if os.getenv('FEATUREGATE_KUBEMARK_NETWORK_PERF', 'false').lower() in ('true', '1'):
+            cmd = "cat /proc/1/sched | head -n 1"
+            r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+            outstr = r.stdout.read().decode().strip()
+            if not "systemd" in outstr:
+                logger.info("Overriding default-gateway/bouncer MAC address for kubemark virtual node")
+                cmd = "ip route show default | awk \'{print $3}\'"
+                r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+                default_gw_ip = r.stdout.read().decode().strip()
+                logger.info("Default gateway IP: {}".format(default_gw_ip))
+                cmd = 'arp ' + f'''{default_gw_ip}''' + ' | grep ' + f'''{default_gw_ip}''' + ' | awk \'{print $3}\''
+                r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+                default_gw_mac = r.stdout.read().decode().strip()
+                logger.info("Default gateway MAC: {}".format(default_gw_mac))
+                dst_mac = default_gw_mac
+                logger.info("Override MAC: {}".format(dst_mac))
         jsonconf = {
             "ep": {
                 "tunnel_id": interface.address.tunnel_id,
@@ -598,6 +615,8 @@ def update_agent_metadata(self, interface):
                 "iface": default_itf
             }
         }
+        if dst_mac != "":
+            jsonconf["dst_mac_override"] = dst_mac
         jsonconf = json.dumps(jsonconf)
         cmd = f'''{self.trn_cli_update_agent_metadata} -i \'{itf}\' -j \'{jsonconf}\''''
         logger.info("update_agent_metadata: {}".format(cmd))

diff --git a/src/cli/trn_cli_agent.c b/src/cli/trn_cli_agent.c
@@ -214,6 +214,7 @@ int trn_cli_update_agent_md_subcmd(CLIENT *clnt, int argc, char *argv[])
 
 	char eth_itf[20];
 	agent_md.eth.interface = eth_itf;
+	memset(agent_md.dst_mac_override, 0, sizeof(agent_md.dst_mac_override));
 	int err = trn_cli_parse_agent_md(json_str, &agent_md);
 	cJSON_Delete(json_str);
 

diff --git a/src/cli/trn_cli_common.c b/src/cli/trn_cli_common.c
@@ -611,11 +611,25 @@ int trn_cli_parse_agent_md(const cJSON *jsonobj,
 	cJSON *ep = cJSON_GetObjectItem(jsonobj, "ep");
 	cJSON *net = cJSON_GetObjectItem(jsonobj, "net");
 	cJSON *eth = cJSON_GetObjectItem(jsonobj, "eth");
+	cJSON *mac_override = cJSON_GetObjectItem(jsonobj, "dst_mac_override");
 	int err_ep, err_net, err_eth;
 	err_ep = trn_cli_parse_ep(ep, &agent_md->ep);
 	err_net = trn_cli_parse_net(net, &agent_md->net);
 	err_eth = trn_cli_parse_tun_intf(eth, &agent_md->eth);
 
+	if (mac_override != NULL && cJSON_IsString(mac_override)) {
+		if (6 == sscanf(mac_override->valuestring,
+				"%hhx:%hhx:%hhx:%hhx:%hhx:%hhx%*c",
+				&agent_md->dst_mac_override[0], &agent_md->dst_mac_override[1],
+				&agent_md->dst_mac_override[2], &agent_md->dst_mac_override[3],
+				&agent_md->dst_mac_override[4], &agent_md->dst_mac_override[5])) {
+		} else {
+			/* invalid mac override */
+			print_err("Error: Invalid MAC Override\n");
+			return -EINVAL;
+		}
+	}
+
 	if (err_ep || err_net || err_eth) {
 		return -EINVAL;
 	}

diff --git a/src/dmn/trn_rpc_protocol_handlers_1.c b/src/dmn/trn_rpc_protocol_handlers_1.c
@@ -1159,6 +1159,11 @@ int *update_agent_md_1_svc(rpc_trn_agent_metadata_t *agent_md,
 	amd.ep.hosted_iface = amd.eth.iface_index;
 	memcpy(amd.ep.mac, agent_md->ep.mac, 6 * sizeof(amd.ep.mac[0]));
 
+	amd.dst_mac_override = 0;
+	unsigned char *dst_mac = (unsigned char *)&amd.dst_mac_override;
+	memcpy(dst_mac, agent_md->dst_mac_override, 6 * sizeof(unsigned char));
+	TRN_LOG_DEBUG("Override MAC: [%lx]", amd.dst_mac_override);
+
 	rc = trn_agent_update_agent_metadata(md, &amd, eth_md);
 
 	if (rc != 0) {

diff --git a/src/include/trn_datamodel.h b/src/include/trn_datamodel.h
@@ -149,6 +149,7 @@ struct agent_metadata_t {
 	struct network_t net;
 	struct endpoint_key_t epkey;
 	struct endpoint_t ep;
+	__u64 dst_mac_override;
 } __attribute__((packed, aligned(4)));
 
 struct ipv4_tuple_t {

diff --git a/src/rpcgen/trn_rpc_protocol.x b/src/rpcgen/trn_rpc_protocol.x
@@ -170,6 +170,7 @@ struct rpc_trn_agent_metadata_t {
        rpc_trn_tun_intf_t eth;
        rpc_trn_endpoint_t ep;
        rpc_trn_network_t net;
+       unsigned char dst_mac_override[6];
 };
 
 enum rpc_trn_pipeline_stage {

diff --git a/src/xdp/trn_agent_xdp.c b/src/xdp/trn_agent_xdp.c
@@ -161,11 +161,15 @@ static __inline int trn_encapsulate(struct transit_packet *pkt,
 		}
 
 		d_mac = r_ep->mac;
-
 	} else {
 		return XDP_DROP;
 	}
 
+	if (metadata->dst_mac_override != 0) {
+		__builtin_memcpy(d_mac, (unsigned char *)&metadata->dst_mac_override,
+			 6 * sizeof(unsigned char));
+	}
+
 	struct packet_metadata_key_t packet_metadata_key;
 	struct packet_metadata_t *packet_metadata;
 	packet_metadata_key.tunip[0] = metadata->epkey.tunip[0];