Add handle function for same-numa option

Check whether the same-numa option enable, if not, will reserve at least one hugepage on each numa. Otherwise, only reserve hugepages on the active numa node. Signed-off-by: Jin Liu <[email protected]> Signed-off-by: Louis Peens <[email protected]>
Netronome · Jun 1, 2022 · f3750eb · f3750eb
1 parent 44880f4
commit f3750eb
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 7 deletions.
diff --git a/dpdk_eal.c b/dpdk_eal.c
@@ -50,7 +50,7 @@ int dpdk_eal_initialize(const struct dpdk_conf *conf)
 	int ret;
 	char buf[32];
 	unsigned i;
-	int l;
+	int len = 0;
 	cpuinfo_t *cpu = get_cpuinfo();
 	uint32_t socket_bitmap = 0;
 	uint32_t numa_bitmap = 0;
@@ -113,16 +113,25 @@ int dpdk_eal_initialize(const struct dpdk_conf *conf)
 	add_arg(&argv, &argc, "-n"); /* Mem channels. */
 	add_arg(&argv, &argc, "4");
 	add_arg(&argv, &argc, "--socket-mem"); /* Memory to allocate per NUMA node (MiB). */
-	l=0;
-	for (i=0; i<cpu->numnodes; i++) {
+
+	/* By default, XVIO reserve at least 1 MiB hugepage memory on each numa,
+	 * as error will occur while use NIC on that numa node but not assign
+	 * hugepage.
+	 * In order to reduce memory use, ensure used NIC and CPUs are on the same
+	 * numa, then set 'conf->enable_same_numa', XVIO will only reserve hugepages
+	 * on the active numa node.
+	 */
+	for (i = 0; i < cpu->numnodes; i++) {
 		if (i >= 1)
-			l += snprintf(buf + l, 32 - l, ",");
-		if ((1<<i) & numa_bitmap)
-			l += snprintf(buf + l, 32 - l,
+			len += snprintf(buf + len, 32 - len, ",");
+
+		if ((1 << i) & numa_bitmap)
+			len += snprintf(buf + len, 32 - len,
 				(conf->use_jumbo || conf->enable_tso) ?
 				"2750" : "768");
 		else
-			l += snprintf(buf + l, 32 - l, "1");
+			len += snprintf(buf + len, 32 - len,
+				conf->enable_same_numa ? "0" : "1");
 	}
 	add_arg(&argv, &argc, buf);
 	optind = 0;

diff --git a/dpdk_eal.h b/dpdk_eal.h
@@ -43,6 +43,7 @@ struct dpdk_conf {
 	uint64_t core_bitmap;
 	unsigned use_jumbo:1;
 	unsigned enable_tso:1;
+	unsigned enable_same_numa:1;
 };
 
 int dpdk_eal_initialize(const struct dpdk_conf *conf);

diff --git a/virtio_forwarder_main.c b/virtio_forwarder_main.c
@@ -606,6 +606,16 @@ cmdline_enable_dynamic_sockets(void *opaque __attribute__((unused)),
 }
 #endif
 
+static int
+cmdline_enable_same_numa(void *opaque __attribute__((unused)),
+			const char *arg __attribute__((unused)),
+			int opt_index __attribute__((unused)))
+{
+	dpdk_cfg.enable_same_numa = 1;
+
+	return 0;
+}
+
 static int configure_signals(void)
 {
 	sigset_t sigset;
@@ -672,6 +682,7 @@ cmdline_opt_t opts[] = {
 #if RTE_VERSION >= RTE_VERSION_NUM(17,5,0,0)
 	{ "dynamic-sockets", 'd', 0, cmdline_enable_dynamic_sockets, 0, "Connect to sockets dynamically instead of creating the default sockets (default: disabled)" },
 #endif
+	{ "same-numa", 'a', 0, cmdline_enable_same_numa, 0, "No longer reserve hugapage on all numa, just reserve hugapage on numa that been used (default: disable)" },
 	{ "version", 'v', CMDLINE_PARAM_FLAG_TERMINATE, cmdline_show_version, 0, "Show version number and exit" },
 	{ 0, 0, 0, 0, 0, "\n\nVirtio-forwarder daemon: forward packets between SR-IOV VFs (serviced by DPDK) and VirtIO network backend.\n" }
 };