From 73b60f314a983e58bcaaac8e1a3aca63adb4d48d Mon Sep 17 00:00:00 2001 From: Saikrishna Edupuganti Date: Wed, 3 Feb 2021 08:45:53 +0000 Subject: [PATCH] Upgrade DPDK to 20.11 - address memseg and mbuf changes - move to meson/ninja build system Signed-off-by: Saikrishna Edupuganti --- bessctl/conf/port/vhost/launch_vm.py | 2 +- build.py | 60 ++++------------------------ container_build.py | 2 +- core/Makefile | 22 ++-------- core/packet.h | 10 +---- core/packet_pool.cc | 6 +-- env/Dockerfile | 2 +- env/build-dep.yml | 12 +++++- 8 files changed, 29 insertions(+), 87 deletions(-) diff --git a/bessctl/conf/port/vhost/launch_vm.py b/bessctl/conf/port/vhost/launch_vm.py index e17b8b692..7417b0eac 100755 --- a/bessctl/conf/port/vhost/launch_vm.py +++ b/bessctl/conf/port/vhost/launch_vm.py @@ -175,7 +175,7 @@ def run_forward(vm_id, num_nics): nics += ' 00:1{nic}.0'.format(nic=i) scp(vm_id, os.path.join(bess_dir, 'bin/dpdk-devbind.py'), '') - scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-19.11.4/build/app/testpmd'), '') + scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-20.11/build/app/dpdk-testpmd'), '') # virtio-pci devices should not be bound to any driver cmd = ssh_cmd(vm_id, 'sudo ./dpdk-devbind.py -u %s' % nics) diff --git a/build.py b/build.py index ff4289523..299a8ca45 100755 --- a/build.py +++ b/build.py @@ -89,14 +89,13 @@ def cmd(cmd, quiet=False, shell=False): DEPS_DIR = '%s/deps' % BESS_DIR DPDK_URL = 'https://fast.dpdk.org/rel' -DPDK_VER = 'dpdk-19.11.4' +DPDK_VER = 'dpdk-20.11' DPDK_TARGET = 'x86_64-native-linuxapp-gcc' kernel_release = cmd('uname -r', quiet=True).strip() DPDK_DIR = '%s/%s' % (DEPS_DIR, DPDK_VER) -DPDK_CFLAGS = '"-g -w"' -DPDK_CONFIG = '%s/build/.config' % DPDK_DIR +DPDK_BUILD = '%s/build' % DPDK_DIR extra_libs = set() cxx_flags = [] @@ -213,46 +212,6 @@ def is_kernel_header_installed(): return os.path.isdir("/lib/modules/%s/build" % kernel_release) -def check_kernel_headers(): - # If kernel header is not available, do not attempt to build - # any components that require kernel. - if not is_kernel_header_installed(): - set_config(DPDK_CONFIG, 'CONFIG_RTE_EAL_IGB_UIO', 'n') - set_config(DPDK_CONFIG, 'CONFIG_RTE_KNI_KMOD', 'n') - set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_KNI', 'n') - set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_PMD_KNI', 'n') - - -def check_bnx(): - if check_header('zlib.h', 'gcc') and check_c_lib('z'): - extra_libs.add('z') - else: - print(' - "zlib1g-dev" is not available. Disabling BNX2X PMD...') - set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_BNX2X_PMD', 'n') - - -def check_mlx(): - if check_header('infiniband/ib.h', 'gcc') and check_c_lib('mlx4') and \ - check_c_lib('mlx5'): - extra_libs.add('ibverbs') - extra_libs.add('mlx4') - extra_libs.add('mlx5') - else: - print(' - "Mellanox OFED" is not available. ' - 'Disabling MLX4 and MLX5 PMDs...') - if check_header('infiniband/verbs.h', 'gcc'): - print(' NOTE: "libibverbs-dev" does exist, but it does not ' - 'work with MLX PMDs. Instead download OFED from ' - 'http://www.melloanox.com') - set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX4_PMD', 'n') - set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX5_PMD', 'n') - - -def generate_dpdk_extra_mk(): - with open('core/extra.dpdk.mk', 'w') as fp: - fp.write('LIBS += %s\n' % ' '.join(['-l' + lib for lib in extra_libs])) - - def find_current_plugins(): "return list of existing plugins" result = [] @@ -293,16 +252,14 @@ def download_dpdk(quiet=False): def configure_dpdk(): print('Configuring DPDK...') - cmd('make -C %s config T=%s' % (DPDK_DIR, DPDK_TARGET)) - - check_kernel_headers() - check_mlx() - generate_dpdk_extra_mk() + meson_opts = '--buildtype=debugoptimized' arch = os.getenv('CPU') if arch: print(' - Building DPDK with -march=%s' % arch) - set_config(DPDK_CONFIG, "CONFIG_RTE_MACHINE", arch) + meson_opts += ' -Dmachine=%s' % arch + + cmd('meson %s %s %s' % (meson_opts, DPDK_BUILD, DPDK_DIR)) def makeflags(): @@ -342,10 +299,7 @@ def build_dpdk(): cmd('patch -d %s -N -p1 < %s || true' % (DPDK_DIR, f), shell=True) print('Building DPDK...') - nproc = int(cmd('nproc', quiet=True)) - cmd('make -C %s EXTRA_CFLAGS=%s %s' % (DPDK_DIR, - DPDK_CFLAGS, - makeflags())) + cmd('ninja -C %s install' % DPDK_BUILD) def generate_protobuf_files(): diff --git a/container_build.py b/container_build.py index d8b4bf3dc..a0b2a8794 100755 --- a/container_build.py +++ b/container_build.py @@ -39,7 +39,7 @@ import re import argparse -IMAGE = 'nefelinetworks/bess_build:latest' + os.getenv('TAG_SUFFIX', '') +IMAGE = 'nefelinetworks/bess_build:' + os.getenv('TAG_SUFFIX', 'latest') BESS_DIR_HOST = os.path.dirname(os.path.abspath(__file__)) BESS_DIR_CONTAINER = '/build/bess' BUILD_SCRIPT = './build.py' diff --git a/core/Makefile b/core/Makefile index cff111737..657417916 100644 --- a/core/Makefile +++ b/core/Makefile @@ -61,28 +61,13 @@ endif HAS_PKG_CONFIG := $(shell command -v $(PKG_CONFIG) 2>&1 >/dev/null && echo yes || echo no) -RTE_SDK ?= $(abspath ../deps/dpdk-19.11.4) -RTE_TARGET ?= $(shell uname -m)-native-linuxapp-gcc -DPDK_LIB ?= dpdk - -ifneq ($(wildcard $(RTE_SDK)/$(RTE_TARGET)/*),) - DPDK_INC_DIR := $(RTE_SDK)/$(RTE_TARGET)/include - DPDK_LIB_DIR := $(RTE_SDK)/$(RTE_TARGET)/build/lib -else ifneq ($(wildcard $(RTE_SDK)/build/*),) - # if the user didn't do "make install" for DPDK - DPDK_INC_DIR := $(RTE_SDK)/build/include - DPDK_LIB_DIR := $(RTE_SDK)/build/lib -else ifneq ($(MAKECMDGOALS),clean) - $(error DPDK is not available. Make sure $(abspath $(RTE_SDK)) is available and built) -endif - # We always want these libraries to be dynamically linked even when the # user requests a static build. ALWAYS_DYN_LIBS := -lpthread -ldl # These libraries are not supported by pkg-config. ALWAYS_LIBS := -lpcap -lgflags -lnuma # If pkg-config is available, we just need a list of the dependecies. -PKG_CONFIG_DEPS := libglog protobuf grpc++ libunwind zlib +PKG_CONFIG_DEPS := libdpdk libglog protobuf grpc++ libunwind zlib # If pkg-config is not available, we need to list the libs we depend on. NO_PKG_CONFIG_LIBS := -lglog -lgflags -lprotobuf -lgrpc++ -lunwind -lz # If pkg-config is not available and we're static linking, we also need @@ -110,7 +95,7 @@ endif COREDIR := $(abspath .) CPU ?= native CXXFLAGS += -std=c++17 -g3 -ggdb3 -march=$(CPU) \ - -isystem $(DPDK_INC_DIR) -isystem $(COREDIR) \ + -isystem $(COREDIR) \ -isystem $(dir $<).. -isystem $(COREDIR)/modules \ -D_GNU_SOURCE \ -Werror -Wall -Wextra -Wcast-align -Wno-error=deprecated-declarations \ @@ -133,7 +118,7 @@ ifeq "$(shell expr $(CXXCOMPILER) = g++)" "1" CXXFLAGS += -fno-gnu-unique endif -LDFLAGS += -rdynamic -L$(DPDK_LIB_DIR) -Wl,-rpath=$(DPDK_LIB_DIR) -pthread +LDFLAGS += -rdynamic ifdef BESS_LINK_DYNAMIC LIBS_ALL_SHARED = -Wl,-call_shared LIBS_DL_SHARED = @@ -158,7 +143,6 @@ else # Used static libraries endif LIBS += -Wl,-non_shared \ - -Wl,--whole-archive -l$(DPDK_LIB) -Wl,--no-whole-archive \ $(LIBS_ALL_SHARED) \ $(PKG_LIBS) $(ALWAYS_LIBS) \ $(LIBS_DL_SHARED) \ diff --git a/core/packet.h b/core/packet.h index 6b9b2ef7b..0471f383c 100644 --- a/core/packet.h +++ b/core/packet.h @@ -261,20 +261,14 @@ class alignas(64) Packet { const uint16_t buf_len_; // offset 56: - uint64_t _dummy6_; // rte_mbuf.timestamp - - // 2nd cacheline - fields only used in slow path or on TX -------------- - // offset 64: - uint64_t _dummy7_; // rte_mbuf.userdata - - // offset 72: struct rte_mempool *pool_; // Pool from which mbuf was allocated. - // offset 80: + // offset 60: Packet *next_; // Next segment. nullptr if not scattered. // offset 88: uint64_t _dummy8; // rte_mbuf.tx_offload + // TODO: Add struct rte_mbuf_ext_shared_info *shinfo; uint16_t _dummy9; // rte_mbuf.priv_size uint16_t _dummy10; // rte_mbuf.timesync uint32_t _dummy11; // rte_mbuf.seqn diff --git a/core/packet_pool.cc b/core/packet_pool.cc index a1d529f02..c453fccd2 100644 --- a/core/packet_pool.cc +++ b/core/packet_pool.cc @@ -241,14 +241,14 @@ DpdkPacketPool::DpdkPacketPool(size_t capacity, int socket_id) static Packet *paddr_to_snb_memchunk(struct rte_mempool_memhdr *chunk, phys_addr_t paddr) { - if (chunk->phys_addr == RTE_BAD_IOVA) { + if (chunk->iova == RTE_BAD_IOVA) { return nullptr; } - if (chunk->phys_addr <= paddr && paddr < chunk->phys_addr + chunk->len) { + if (chunk->iova <= paddr && paddr < chunk->iova + chunk->len) { uintptr_t vaddr; - vaddr = (uintptr_t)chunk->addr + paddr - chunk->phys_addr; + vaddr = (uintptr_t)chunk->addr + paddr - chunk->iova; return reinterpret_cast(vaddr); } diff --git a/env/Dockerfile b/env/Dockerfile index cc289480b..e1553786b 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -29,7 +29,7 @@ ARG BESS_DPDK_BRANCH=master RUN cd /build/bess && \ curl -s -L https://github.com/NetSys/bess/archive/${BESS_DPDK_BRANCH}.tar.gz | tar zx --strip-components=1 && \ ./build.py dpdk && \ - cp /build/bess/deps/dpdk-19.11.4/build/app/testpmd /usr/local/bin/ && \ + cp /build/bess/deps/dpdk-20.11/build/app/dpdk-testpmd /usr/local/bin/ && \ rm -rf /build/bess ENV CCACHE_DIR=/tmp/ccache diff --git a/env/build-dep.yml b/env/build-dep.yml index 5f9af1f2f..431e224a3 100644 --- a/env/build-dep.yml +++ b/env/build-dep.yml @@ -23,9 +23,19 @@ - libgflags-dev - libgoogle-glog-dev - libgtest-dev - - python + - python3 + - python3-pip + - python3-setuptools - pkg-config + - name: Install DPDK build system + become: true + pip: + name: + - meson + - ninja + executable: pip3 + # pre-packaged meat for Bionic Beaver or higher - name: Install gRPC and its requirements (apt) apt: name={{item}}