From c35c14a2affc409ba6211782dcc039cc5ba8d239 Mon Sep 17 00:00:00 2001 From: Chenming C Date: Fri, 17 Jun 2022 09:49:52 +0800 Subject: [PATCH] feat(cpufreq): add a definition to disable lock cpu frequency feature --- README-ZH.md | 12 +- README.md | 10 +- src/main.c | 301 +++++++++++++++++++++++++-------------------------- src/setup.c | 3 +- 4 files changed, 169 insertions(+), 157 deletions(-) diff --git a/README-ZH.md b/README-ZH.md index b4b8884..758dea2 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -1,5 +1,7 @@ # smart_offload +[![Build](https://github.com/chen622/smart_offload/actions/workflows/build.yml/badge.svg)](https://github.com/chen622/smart_offload/actions/workflows/build.yml) + ## 一、主要功能 - 将流经的数据包上送到处理线程中，并在第 n 个包后下发对应的流表进行卸载 @@ -24,7 +26,7 @@ ## 四、快速上手 ### 配置要求： -- DPDK 20.11.3.0.4 +- DPDK 20.11.5 - 配备 Mellanox/Nvidia 智能网卡的 x86 服务器 ### 运行指令 @@ -34,4 +36,10 @@ cmake .. make # -l 用于指定核心 -a 用于指定网口 ./smart_offload -l 1-9 -a 82:00.0 -``` \ No newline at end of file +``` + +## 五、问题 + +### Unable to set Power Management Environment + +无法锁定 CPU 的运行频率，可能是由于缺乏相关权限。可以通过增加一个定义`-DVM=true`来关闭此功能 \ No newline at end of file diff --git a/README.md b/README.md index 464c6ae..ead34b8 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ The diagram is written with Chinese, you can see the English comment of function ## 3. Quick Start ### Requirements： -- DPDK 20.11.3.0.4 -- x86 server with Mellanox/Nvidia smart-nic. +- DPDK 20.11.5 +- x86 server with Mellanox/Nvidia SmartNic. ### Run ```bash @@ -29,3 +29,9 @@ make # -l Specify the core; -a Specify the port ./smart_offload -l 1-9 -a 82:00.0 ``` + +## 4. Questions + +### Unable to set Power Management Environment + +Can not set frequency of CPU cores, you can a definition `-DVM=true` to disable this feature. diff --git a/src/main.c b/src/main.c index dcb711e..e701536 100644 --- a/src/main.c +++ b/src/main.c @@ -35,168 +35,165 @@ char *zlog_conf = "conf/zlog.conf"; #endif static void signal_handler(int signum) { - if (signum == SIGINT || signum == SIGTERM) { - dzlog_info("Signal %d received, preparing to exit...", - signum); - force_quit = true; - } + if (signum == SIGINT || signum == SIGTERM) { + dzlog_info("Signal %d received, preparing to exit...", + signum); + force_quit = true; + } } - int main(int argc, char **argv) { - - - /* General return value */ - int ret; - char err_msg[MAX_ERROR_MESSAGE_LENGTH]; - /* Quantity of ports */ - uint16_t port_quantity; - /* Quantity of slave works */ - uint16_t worker_quantity; - uint16_t port_id = 0; - - /* Setup environment of DPDK */ - ret = rte_eal_init(argc, argv); - if (ret < 0) { - smto_exit(EXIT_FAILURE, "invalid EAL arguments"); - } - - /* Setup zlog */ - ret = dzlog_init(zlog_conf, "main"); - if (ret) { - smto_exit(EXIT_FAILURE, "zlog init failed"); - } - - /* Check the instruction */ + /* General return value */ + int ret; + char err_msg[MAX_ERROR_MESSAGE_LENGTH]; + /* Quantity of ports */ + uint16_t port_quantity; + /* Quantity of slave works */ + uint16_t worker_quantity; + uint16_t port_id = 0; + + /* Setup environment of DPDK */ + ret = rte_eal_init(argc, argv); + if (ret < 0) { + smto_exit(EXIT_FAILURE, "invalid EAL arguments"); + } + + /* Setup zlog */ + ret = dzlog_init(zlog_conf, "main"); + if (ret) { + smto_exit(EXIT_FAILURE, "zlog init failed"); + } + + /* Check the instruction */ #if defined(__SSE2__) - dzlog_debug("Find SSE2 support"); + dzlog_debug("Find SSE2 support"); #else #error No vector engine (SSE, NEON, ALTIVEC) available, check your toolchain #endif - /* Listen to the shutdown event */ - force_quit = false; - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - dzlog_debug("Running on socket #%d", rte_socket_id()); - - /* Check the quantity of network ports */ - port_quantity = rte_eth_dev_count_avail(); - if (port_quantity < 1) { - smto_exit(EXIT_FAILURE, "no enough Ethernet ports found"); - } else if (port_quantity > 2) { - dzlog_warn("%d ports detected, but we only use two", port_quantity); - } - - /* Check the quantity of workers */ - worker_quantity = rte_lcore_count(); - if (worker_quantity != GENERAL_QUEUES_QUANTITY + 1) { - snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, - "worker quantity does not match the queue quantity, it should be %u rather than %u", - GENERAL_QUEUES_QUANTITY + 1, worker_quantity); - smto_exit(EXIT_FAILURE, err_msg); - } - - /* Initialize the memory pool of dpdk */ - struct rte_mempool *mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NUM_MBUFS, CACHE_SIZE, 0, - RTE_MBUF_DEFAULT_BUF_SIZE, - rte_socket_id()); - if (mbuf_pool == NULL) { - smto_exit(EXIT_FAILURE, "cannot init mbuf pool"); + /* Listen to the shutdown event */ + force_quit = false; + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + dzlog_debug("Running on socket #%d", rte_socket_id()); + + /* Check the quantity of network ports */ + port_quantity = rte_eth_dev_count_avail(); + if (port_quantity < 1) { + smto_exit(EXIT_FAILURE, "no enough Ethernet ports found"); + } else if (port_quantity > 2) { + dzlog_warn("%d ports detected, but we only use two", port_quantity); + } + + /* Check the quantity of workers */ + worker_quantity = rte_lcore_count(); + if (worker_quantity != GENERAL_QUEUES_QUANTITY + 1) { + snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, + "worker quantity does not match the queue quantity, it should be %u rather than %u", + GENERAL_QUEUES_QUANTITY + 1, worker_quantity); + smto_exit(EXIT_FAILURE, err_msg); + } + + /* Initialize the memory pool of dpdk */ + struct rte_mempool *mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NUM_MBUFS, CACHE_SIZE, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + if (mbuf_pool == NULL) { + smto_exit(EXIT_FAILURE, "cannot init mbuf pool"); + } + + /* Config port and setup hairpin mode */ + if (port_quantity == 1) { + dzlog_debug("ONE port hairpin mode"); + port_id = rte_eth_find_next_owned_by(0, RTE_ETH_DEV_NO_OWNER); + init_port(port_id, mbuf_pool); + setup_one_port_hairpin(port_id); + } else { + dzlog_debug("TWO port hairpin mode unsupported"); + smto_exit(EXIT_FAILURE, "WO port hairpin mode unsupported"); + /* Initialize the network port and do some configure */ + RTE_ETH_FOREACH_DEV(port_id) { + init_port(port_id, mbuf_pool); } - - /* Config port and setup hairpin mode */ - if (port_quantity == 1) { - dzlog_debug("ONE port hairpin mode"); - port_id = rte_eth_find_next_owned_by(0, RTE_ETH_DEV_NO_OWNER); - init_port(port_id, mbuf_pool); - setup_one_port_hairpin(port_id); - } else { - dzlog_debug("TWO port hairpin mode unsupported"); - smto_exit(EXIT_FAILURE, "WO port hairpin mode unsupported"); - /* Initialize the network port and do some configure */ - RTE_ETH_FOREACH_DEV(port_id) { - init_port(port_id, mbuf_pool); - } - setup_two_port_hairpin(); - } - - - struct rte_flow *flow = 0; - struct rte_flow_error flow_error = {0}; - flow = create_default_jump_flow(port_id, &flow_error); - if (flow == NULL) { - snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, - "the default jump flow create failed: %s", flow_error.message); - smto_exit(EXIT_FAILURE, err_msg); - } - flow = create_default_rss_flow(port_id, GENERAL_QUEUES_QUANTITY, &flow_error); - if (flow == NULL) { - snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, - "the default rss flow create failed: %s", flow_error.message); - smto_exit(EXIT_FAILURE, err_msg); - } - - /* Create flow hash map */ - struct rte_hash_parameters flow_hash_map_parameter = { - .name = "flow_hash_table", - .entries = MAX_HASH_ENTRIES, - .key_len = sizeof(union ipv4_5tuple_host), - .hash_func = ipv4_hash_crc, - .hash_func_init_val = 0, - .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF - }; - struct rte_hash *flow_hash_map = rte_hash_create(&flow_hash_map_parameter); - if (flow_hash_map == NULL) { - snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, "unable to create flow hash table"); - smto_exit(EXIT_FAILURE, err_msg); - } - - if (register_aged_event(port_id, flow_hash_map)) { - smto_exit(EXIT_FAILURE, "cannot register from age timeout event"); - } - - uint16_t lcore_id = 0; - uint16_t index = 0; - struct worker_parameter worker_params[GENERAL_QUEUES_QUANTITY]; - RTE_LCORE_FOREACH_WORKER(lcore_id) { - worker_params[index].port_id = port_id; - worker_params[index].queue_id = index; - worker_params[index].flow_hash_map = flow_hash_map; - rte_power_init(lcore_id); - ret = rte_power_set_freq(lcore_id, 2); - if (ret < 0) { - dzlog_warn("worker #%u does not running at the fixed frequency", lcore_id); - } - rte_eal_remote_launch(process_loop, &worker_params[index], lcore_id); - index++; + setup_two_port_hairpin(); + } + + struct rte_flow *flow = 0; + struct rte_flow_error flow_error = {0}; + flow = create_default_jump_flow(port_id, &flow_error); + if (flow == NULL) { + snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, + "the default jump flow create failed: %s", flow_error.message); + smto_exit(EXIT_FAILURE, err_msg); + } + flow = create_default_rss_flow(port_id, GENERAL_QUEUES_QUANTITY, &flow_error); + if (flow == NULL) { + snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, + "the default rss flow create failed: %s", flow_error.message); + smto_exit(EXIT_FAILURE, err_msg); + } + + /* Create flow hash map */ + struct rte_hash_parameters flow_hash_map_parameter = { + .name = "flow_hash_table", + .entries = MAX_HASH_ENTRIES, + .key_len = sizeof(union ipv4_5tuple_host), + .hash_func = ipv4_hash_crc, + .hash_func_init_val = 0, + .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF + }; + struct rte_hash *flow_hash_map = rte_hash_create(&flow_hash_map_parameter); + if (flow_hash_map == NULL) { + snprintf(err_msg, MAX_ERROR_MESSAGE_LENGTH, "unable to create flow hash table"); + smto_exit(EXIT_FAILURE, err_msg); + } + + if (register_aged_event(port_id, flow_hash_map)) { + smto_exit(EXIT_FAILURE, "cannot register from age timeout event"); + } + + uint16_t lcore_id = 0; + uint16_t index = 0; + struct worker_parameter worker_params[GENERAL_QUEUES_QUANTITY]; + RTE_LCORE_FOREACH_WORKER(lcore_id) { + worker_params[index].port_id = port_id; + worker_params[index].queue_id = index; + worker_params[index].flow_hash_map = flow_hash_map; +#ifndef VM + rte_power_init(lcore_id); + ret = rte_power_set_freq(lcore_id, 2); +#endif + if (ret < 0) { + dzlog_warn("worker #%u does not running at the fixed frequency", lcore_id); } - - - rte_eal_mp_wait_lcore(); - - - /* free the memory of flow metadata and the flow hash map*/ - int32_t key_count = rte_hash_count(flow_hash_map); - dzlog_debug("%d flow keys has been added into flow hash map", key_count); - if (key_count > 0) { - char format_key[MAX_ERROR_MESSAGE_LENGTH]; - const void *key = 0; - void *data = 0; - uint32_t *next = 0; - uint32_t current = rte_hash_iterate(flow_hash_map, &key, &data, next); - for (; current != -ENOENT; current = rte_hash_iterate(flow_hash_map, &key, &data, next)) { - int32_t del_key = rte_hash_del_key(flow_hash_map, key); - ret = rte_hash_free_key_with_position(flow_hash_map, del_key); - if (ret) { - dump_pkt_info((union ipv4_5tuple_host *) key, -1, format_key, MAX_ERROR_MESSAGE_LENGTH); - dzlog_error("cannot free hash key(%s)", format_key); - } - rte_free(data); - } + rte_eal_remote_launch(process_loop, &worker_params[index], lcore_id); + index++; + } + + rte_eal_mp_wait_lcore(); + + + /* free the memory of flow metadata and the flow hash map*/ + int32_t key_count = rte_hash_count(flow_hash_map); + dzlog_debug("%d flow keys has been added into flow hash map", key_count); + if (key_count > 0) { + char format_key[MAX_ERROR_MESSAGE_LENGTH]; + const void *key = 0; + void *data = 0; + uint32_t *next = 0; + uint32_t current = rte_hash_iterate(flow_hash_map, &key, &data, next); + for (; current != -ENOENT; current = rte_hash_iterate(flow_hash_map, &key, &data, next)) { + int32_t del_key = rte_hash_del_key(flow_hash_map, key); + ret = rte_hash_free_key_with_position(flow_hash_map, del_key); + if (ret) { + dump_pkt_info((union ipv4_5tuple_host *) key, -1, format_key, MAX_ERROR_MESSAGE_LENGTH); + dzlog_error("cannot free hash key(%s)", format_key); + } + rte_free(data); } - rte_hash_free(flow_hash_map); + } + rte_hash_free(flow_hash_map); - smto_exit(EXIT_SUCCESS, "SUCCESS! All core stop running!"); + smto_exit(EXIT_SUCCESS, "SUCCESS! All core stop running!"); } diff --git a/src/setup.c b/src/setup.c index 23c583c..020960f 100644 --- a/src/setup.c +++ b/src/setup.c @@ -142,11 +142,12 @@ void smto_exit(int exit_code, const char *format) { rte_eth_dev_stop(port_id); rte_eth_dev_close(port_id); } - +#ifndef VM uint16_t lcore_id = 0; RTE_LCORE_FOREACH_WORKER(lcore_id) { rte_power_exit(lcore_id); } +#endif /* clean up the EAL */ rte_eal_cleanup();