Skip to content

Commit 869e297

Browse files
committed
Use bpftool instead of iproute2 to load eBPF and XDP maps.
Signed-off-by: fruffy <[email protected]>
1 parent 947469d commit 869e297

12 files changed

+119
-68
lines changed

backends/ebpf/CMakeLists.txt

+4-4
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,17 @@ if(NOT APPLE)
2121
set(FETCHCONTENT_QUIET OFF)
2222
fetchcontent_declare(
2323
bpfrepo
24-
URL https://github.com/libbpf/libbpf/archive/refs/tags/v1.4.1.tar.gz
25-
URL_HASH SHA256=cc01a3a05d25e5978c20be7656f14eb8b6fcb120bb1c7e8041e497814fc273cb
26-
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/contrib/libbpf
24+
URL https://github.com/libbpf/bpftool/releases/download/v7.5.0/bpftool-libbpf-v7.5.0-sources.tar.gz
25+
# URL_HASH SHA256=cc01a3a05d25e5978c20be7656f14eb8b6fcb120bb1c7e8041e497814fc273cb
26+
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/contrib/bpftool
2727
USES_TERMINAL_DOWNLOAD TRUE
2828
GIT_PROGRESS TRUE
2929
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
3030
)
3131
fetchcontent_makeavailable(bpfrepo)
3232
set(FETCHCONTENT_QUIET ${FETCHCONTENT_QUIET_PREV})
3333
# Check if we have already built the libbpf library.
34-
find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/")
34+
find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/install/libbpf/")
3535
if (NOT LIBBPF)
3636
message("Building libbpf...")
3737
execute_process(

backends/ebpf/build_libbpf

+15-12
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1111
# See the License for the specific language governing permissions and
1212
# limitations under the License.
13-
""" This programs builds a libbpf static library and places it in the runtime
13+
""" This programs builds a bpftool binary and the libbpf static library and places it in the runtime
1414
folder. The library and its headers are required by the kernel target.
1515
"""
1616

@@ -31,21 +31,24 @@ def main() -> int:
3131
level=logging.WARN,
3232
filemode="w",
3333
)
34-
libbpf_dir = FILE_DIR.joinpath("runtime/contrib/libbpf")
35-
libbpf_src_dir = libbpf_dir.joinpath("src")
36-
libbpf_build_dir = libbpf_src_dir.joinpath("build")
37-
libbpf_target_dir = FILE_DIR.joinpath("runtime")
38-
# Create the libbpf build directory
39-
testutils.check_and_create_dir(libbpf_build_dir)
40-
# Build libbpf
41-
mk_cmd = f"make -C {libbpf_src_dir} install "
34+
bpftool_dir = FILE_DIR.joinpath("runtime/contrib/bpftool")
35+
bpftool_src_dir = bpftool_dir.joinpath("src")
36+
bpftool_build_dir = bpftool_src_dir.joinpath("build")
37+
bpftool_target_dir = FILE_DIR.joinpath("runtime/install/")
38+
bpftool_target_dir.mkdir(parents=True, exist_ok=True)
39+
# Create the bpftool build directory
40+
testutils.check_and_create_dir(bpftool_build_dir)
41+
# Build bpftool
42+
mk_cmd = f"make -C {bpftool_src_dir} install "
43+
mk_cmd += f"OUTPUT={bpftool_target_dir}/ "
4244
mk_cmd += "BUILD_STATIC_ONLY=y "
43-
mk_cmd += f"OBJDIR={libbpf_build_dir} "
44-
mk_cmd += f"DESTDIR={libbpf_target_dir} "
45+
mk_cmd += f"OBJDIR={bpftool_build_dir} "
46+
mk_cmd += f"DESTDIR={bpftool_target_dir} "
4547
mk_cmd += "EXTRA_CFLAGS=-fPIE"
48+
print(mk_cmd)
4649
result = testutils.exec_process(args=mk_cmd)
4750
if result.returncode != testutils.SUCCESS:
48-
testutils.log.error("Could not build libbpf")
51+
testutils.log.error("Could not build bpftool")
4952
return result.returncode
5053

5154

backends/ebpf/ebpfProgram.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ void EBPFProgram::emitC(CodeBuilder *builder, const std::filesystem::path &heade
102102
if (model.arch == ModelArchitecture::XdpSwitch)
103103
builder->target->emitCodeSection(builder, "xdp"_cs);
104104
else
105-
builder->target->emitCodeSection(builder, "prog"_cs);
105+
builder->target->emitCodeSection(builder, "tc"_cs);
106106
builder->emitIndent();
107107
builder->target->emitMain(builder, functionName, model.CPacketName.toString());
108108
builder->blockStart();

backends/ebpf/run-ebpf-test.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,9 @@ def __init__(self):
123123
self.replace = False # Replace previous outputs.
124124
self.target = "test" # The name of the target compiler.
125125
# Actual location of the test framework.
126-
self.testdir = str(FILE_DIR)
126+
self.testdir = FILE_DIR
127127
# The location of the eBPF runtime, some targets may overwrite this.
128-
self.runtimedir = str(FILE_DIR.joinpath("runtime"))
128+
self.runtimedir = FILE_DIR.joinpath("runtime")
129129
self.extern = "" # Path to C file with extern definition.
130130

131131

backends/ebpf/runtime/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
install
+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
libbpf
2+
bpftool

backends/ebpf/runtime/ebpf_kernel.h

+29-26
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ limitations under the License.
2323

2424
#include "ebpf_common.h"
2525

26-
#include <bpf/bpf_endian.h> // definitions for bpf_ntohs etc...
26+
#include "install/libbpf/include/bpf/bpf_endian.h" // definitions for bpf_ntohs etc...
2727

2828
#undef htonl
2929
#undef htons
@@ -47,7 +47,7 @@ limitations under the License.
4747

4848
#ifdef CONTROL_PLANE // BEGIN EBPF USER SPACE DEFINITIONS
4949

50-
#include <bpf/bpf.h> // bpf_obj_get/pin, bpf_map_update_elem
50+
#include "install/libbpf/include/bpf/bpf.h" // bpf_obj_get/pin, bpf_map_update_elem
5151

5252
#define BPF_USER_MAP_UPDATE_ELEM(index, key, value, flags)\
5353
bpf_map_update_elem(index, key, value, flags)
@@ -56,30 +56,18 @@ limitations under the License.
5656

5757
#else // BEGIN EBPF KERNEL DEFINITIONS
5858

59-
#include <linux/pkt_cls.h> // TC_ACT_OK, TC_ACT_SHOT
60-
#include "linux/bpf.h" // types, and general bpf definitions
59+
// These files are provided by the system, not libbpf.
60+
#include "contrib/bpftool/include/uapi/linux/bpf.h" // BPF_ANY,
61+
#include "contrib/bpftool/include/uapi/linux/pkt_cls.h" // TC_ACT_OK, TC_ACT_SHOT
6162
// This file contains the definitions of all the kernel bpf essentials
62-
#include <bpf/bpf_helpers.h>
63-
64-
/// A helper structure used by an eBPF C program
65-
/// to describe map attributes for the elf_bpf loader
66-
/// FIXME: We only need this because we are loading with iproute2
67-
struct bpf_elf_map {
68-
__u32 type;
69-
__u32 size_key;
70-
__u32 size_value;
71-
__u32 max_elem;
72-
__u32 flags;
73-
__u32 id;
74-
__u32 pinning;
75-
__u32 inner_id;
76-
__u32 inner_idx;
77-
};
63+
#include "install/libbpf/include/bpf/bpf_helpers.h"
64+
#include "install/libbpf/include/bpf/btf.h"
7865

7966
/// Simple descriptor which replaces the kernel sk_buff structure.
8067
#define SK_BUFF struct __sk_buff
8168

8269
/// From iproute2, annotate table with BTF which allows to read types at runtime.
70+
/// TODO: Do we need this with Ubuntu 24.04?
8371
#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \
8472
struct ____btf_map_##name { \
8573
type_key key; \
@@ -90,7 +78,22 @@ struct bpf_elf_map {
9078
____btf_map_##name = {};
9179

9280
#define REGISTER_START()
93-
#ifndef BTF
81+
/// TODO: Do we need this with Ubuntu 24.04?
82+
#if !defined(BTF) && (defined(__clang__) && (__clang_major__ < 12))
83+
/// A helper structure used by an eBPF C program
84+
/// to describe map attributes for the elf_bpf loader
85+
/// FIXME: We only need this because we are loading with iproute2. Remove with Ubuntu 24.04
86+
struct bpf_elf_map {
87+
__u32 type;
88+
__u32 size_key;
89+
__u32 size_value;
90+
__u32 max_elem;
91+
__u32 flags;
92+
__u32 id;
93+
__u32 pinning;
94+
__u32 inner_id;
95+
__u32 inner_idx;
96+
};
9497
/// Note: pinning exports the table name globally, do not remove.
9598
#define REGISTER_TABLE(NAME, TYPE, KEY_TYPE, VALUE_TYPE, MAX_ENTRIES) \
9699
struct bpf_elf_map SEC("maps") NAME = { \
@@ -139,7 +142,7 @@ struct { \
139142
VALUE_TYPE *value; \
140143
__uint(max_entries, MAX_ENTRIES); \
141144
__uint(pinning, LIBBPF_PIN_BY_NAME); \
142-
} NAME SEC(".maps");
145+
} NAME SEC(MAPS_ELF_SEC);
143146
#define REGISTER_TABLE_FLAGS(NAME, TYPE, KEY_TYPE, VALUE_TYPE, MAX_ENTRIES, FLAGS) \
144147
struct { \
145148
__uint(type, TYPE); \
@@ -148,14 +151,14 @@ struct { \
148151
__uint(max_entries, MAX_ENTRIES); \
149152
__uint(pinning, LIBBPF_PIN_BY_NAME); \
150153
__uint(map_flags, FLAGS); \
151-
} NAME SEC(".maps");
154+
} NAME SEC(MAPS_ELF_SEC);
152155
#define REGISTER_TABLE_INNER(NAME, TYPE, KEY_TYPE, VALUE_TYPE, MAX_ENTRIES, ID, INNER_IDX) \
153156
struct NAME { \
154157
__uint(type, TYPE); \
155158
KEY_TYPE *key; \
156159
VALUE_TYPE *value; \
157160
__uint(max_entries, MAX_ENTRIES); \
158-
} NAME SEC(".maps");
161+
} NAME SEC(MAPS_ELF_SEC);
159162
#define REGISTER_TABLE_OUTER(NAME, TYPE, KEY_TYPE, VALUE_TYPE, MAX_ENTRIES, INNER_ID, INNER_NAME) \
160163
struct { \
161164
__uint(type, TYPE); \
@@ -164,15 +167,15 @@ struct { \
164167
__uint(max_entries, MAX_ENTRIES); \
165168
__uint(pinning, LIBBPF_PIN_BY_NAME); \
166169
__array(values, struct INNER_NAME); \
167-
} NAME SEC(".maps");
170+
} NAME SEC(MAPS_ELF_SEC);
168171
#define REGISTER_TABLE_NO_KEY_TYPE(NAME, TYPE, KEY_SIZE, VALUE_TYPE, MAX_ENTRIES) \
169172
struct { \
170173
__uint(type, TYPE); \
171174
__uint(key_size, KEY_SIZE); \
172175
VALUE_TYPE *value; \
173176
__uint(max_entries, MAX_ENTRIES); \
174177
__uint(pinning, LIBBPF_PIN_BY_NAME); \
175-
} NAME SEC(".maps");
178+
} NAME SEC(MAPS_ELF_SEC);
176179
#endif
177180
#define REGISTER_END()
178181

backends/ebpf/runtime/kernel.mk

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
33
# Argument for the CLANG compiler
44
LLC ?= llc
55
CLANG ?= clang
6-
override INCLUDES+= -I$(ROOT_DIR) -I$(ROOT_DIR)usr/include/ -I$(ROOT_DIR)contrib/libbpf/include/uapi/
6+
override INCLUDES+= -I$(ROOT_DIR)
77
override LIBS+=
88
# Optimization flags to save space
99
override CFLAGS+= -O2 -g -c -D__KERNEL__ -D__ASM_SYSREG_H \

backends/ebpf/targets/ebpfenv.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ def ns_proc_append(self, proc: subprocess.Popen, cmd: str) -> int:
9696
def ns_proc_close(self, proc: subprocess.Popen, **extra_args: Any) -> int:
9797
"""Close and actually run the process in the namespace. Returns the
9898
exit code."""
99-
testutils.log.info("Executing command: %s", proc)
99+
testutils.log.info("Executing command: %s", proc.args)
100100
result = testutils.run_process(proc, timeout=testutils.TIMEOUT, **extra_args)
101101
if result.returncode != testutils.SUCCESS:
102102
testutils.log.error(
@@ -109,7 +109,7 @@ def _configure_bridge(self, br_name: str) -> int:
109109
avoid ICMPv6 spam."""
110110
# We do not care about failures here
111111
self.ns_exec(f"ip link set dev {br_name} up")
112-
self.ns_exec(f"ip link set dev {br_name} mtu 9000")
112+
self.ns_exec(f"ip link set dev {br_name} mtu 1500")
113113
# Prevent the broadcasting of ipv6 link discovery messages
114114
self.ns_exec("sysctl -w net.ipv6.conf.all.disable_ipv6=1")
115115
self.ns_exec("sysctl -w net.ipv6.conf.default.disable_ipv6=1")
@@ -130,7 +130,7 @@ def _configure_bridge_port(self, port_name: str) -> int:
130130
result = self.ns_exec(cmd)
131131
if result != testutils.SUCCESS:
132132
return result
133-
cmd = f"ip link set dev {port_name} mtu 9000"
133+
cmd = f"ip link set dev {port_name} mtu 1500"
134134
return self.ns_exec(cmd)
135135

136136
def attach_interfaces(self, num_ifaces: int) -> int:

backends/ebpf/targets/kernel_target.py

+60-17
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import time
2121
from glob import glob
2222
from pathlib import Path
23+
from typing import Optional
2324

2425
from .ebpfenv import Bridge
2526
from .target import EBPFTarget
@@ -33,10 +34,15 @@
3334

3435

3536
class Target(EBPFTarget):
36-
EBPF_MAP_PATH = "/sys/fs/bpf/tc/globals"
37+
EBPF_PATH = Path("/sys/fs/bpf")
3738

3839
def __init__(self, tmpdir, options, template):
3940
EBPFTarget.__init__(self, tmpdir, options, template)
41+
self.bpftool = self.runtimedir.joinpath("install/bpftool")
42+
if self.options.target == "xdp":
43+
self.ebpf_map_path = self.EBPF_PATH.joinpath(f"xdp/globals")
44+
else:
45+
self.ebpf_map_path = self.EBPF_PATH.joinpath(f"tc/globals")
4046

4147
def compile_dataplane(self):
4248
# Use clang to compile the generated C code to a LLVM IR
@@ -68,30 +74,33 @@ def _create_runtime(self):
6874
args += "CFLAGS+=-DCONTROL_PLANE "
6975
# add the folder local to the P4 file to the list of includes
7076
args += f"INCLUDES+=-I{os.path.dirname(self.options.p4filename)} "
71-
# some kernel specific includes for libbpf
72-
args += f"INCLUDES+=-I{self.runtimedir}/usr/include "
73-
args += f"INCLUDES+=-I{self.runtimedir}/contrib/libbpf/include/uapi "
74-
args += f"LIBS+={self.runtimedir}/usr/lib64/libbpf.a "
77+
args += f"LIBS+={self.runtimedir}/install/libbpf/libbpf.a "
7578
args += "LIBS+=-lz "
7679
args += "LIBS+=-lelf "
7780
result = testutils.exec_process(args)
7881
if result.returncode != testutils.SUCCESS:
7982
testutils.log.error("Failed to build the filter")
8083
return result.returncode
8184

82-
def _create_bridge(self):
85+
def _create_bridge(self) -> Optional[Bridge]:
8386
# The namespace is the id of the process
8487
namespace = str(os.getpid())
8588
# Number of input files
8689
direction = "in"
8790
num_files = len(glob(self.filename("*", direction)))
8891
# Create the namespace and the bridge with all its ports
89-
br = Bridge(namespace)
90-
result = br.create_virtual_env(num_files)
92+
bridge = Bridge(namespace)
93+
result = bridge.create_virtual_env(num_files)
9194
if result != testutils.SUCCESS:
92-
br.ns_del()
95+
bridge.ns_del()
9396
return None
94-
return br
97+
if self.options.target != "xdp":
98+
# Add the qdisc. MUST be clsact layer.
99+
for port_name in bridge.edge_ports:
100+
result = bridge.ns_exec(f"tc qdisc add dev {port_name} clsact")
101+
if result != testutils.SUCCESS:
102+
return None
103+
return bridge
95104

96105
def _get_run_cmd(self):
97106
direction = "in"
@@ -108,7 +117,7 @@ def _get_run_cmd(self):
108117
cmd += "-d"
109118
return cmd
110119

111-
def _kill_processes(self, procs):
120+
def _kill_processes(self, procs) -> None:
112121
for proc in procs:
113122
# kill process, 15 is SIGTERM
114123
os.kill(proc.pid, 15)
@@ -127,7 +136,7 @@ def _load_filter(self, bridge, proc, port_name):
127136
bridge.ns_exec(f"tc qdisc add dev {port_name} clsact")
128137
cmd = (
129138
f"tc filter add dev {port_name} egress"
130-
f" bpf da obj {self.template}.o section prog verbose"
139+
f" bpf da obj {self.template}.o section tc verbose "
131140
)
132141
return bridge.ns_proc_write(proc, cmd)
133142

@@ -145,6 +154,44 @@ def _attach_filters(self, bridge, proc):
145154
return result
146155
return testutils.SUCCESS
147156

157+
# def _attach_filters(self, bridge: Bridge, proc: subprocess.Popen) -> int:
158+
# # Is this a XDP or TC (ebpf_filter) program?
159+
# p_result = testutils.exec_process(f"objdump -hj xdp {self.template}.o").returncode
160+
# is_xdp = p_result == testutils.SUCCESS
161+
# # Load the specified eBPF object to "port_name" egress
162+
# # As a side-effect, this may create maps in /sys/fs/bpf/
163+
# # Get the command to load eBPF code to all the attached ports
164+
# result = bridge.ns_proc_write(proc, f"mount -t bpf none {self.EBPF_PATH}")
165+
# if result != testutils.SUCCESS:
166+
# return result
167+
# result = bridge.ns_proc_append(proc, f"mkdir -p {self.ebpf_map_path}")
168+
# if result != testutils.SUCCESS:
169+
# return result
170+
# load_type = "xdp" if is_xdp else "tc"
171+
# cmd = f"{self.bpftool} prog load {self.template}.o {self.EBPF_PATH}/ebpf_filter pinmaps {self.ebpf_map_path} type {load_type}"
172+
# result = bridge.ns_proc_append(proc, cmd)
173+
# if result != testutils.SUCCESS:
174+
# return result
175+
176+
# attach_type = "xdp" if is_xdp else "tcx_egress"
177+
# ports = bridge.br_ports if is_xdp else bridge.edge_ports
178+
# if len(ports) > 0:
179+
# for port_name in ports:
180+
# result = bridge.ns_proc_append(
181+
# proc,
182+
# f"{self.bpftool} net attach {attach_type} pinned {self.EBPF_PATH}/ebpf_filter dev {port_name}",
183+
# )
184+
# if result != testutils.SUCCESS:
185+
# return result
186+
# else:
187+
# # No ports attached (no pcap files), load to bridge instead
188+
# result = bridge.ns_proc_append(
189+
# proc,
190+
# f"{self.bpftool} net attach {attach_type} pinned {self.EBPF_PATH}/ebpf_filter dev {bridge.br_name}",
191+
# )
192+
193+
# return result
194+
148195
def _run_tcpdump(self, bridge, filename, port):
149196
cmd = f"{bridge.get_ns_prefix()} tcpdump -w {filename} -i {port}"
150197
return subprocess.Popen(cmd.split())
@@ -166,14 +213,10 @@ def _run_in_namespace(self, bridge):
166213
return testutils.FAILURE
167214
dump_procs = self._init_tcpdump_listeners(bridge)
168215
result = self._attach_filters(bridge, proc)
169-
if result != testutils.SUCCESS:
170-
return result
171-
# Check if eBPF maps have actually been created
172-
result = bridge.ns_proc_write(proc, f"ls -1 {self.EBPF_MAP_PATH}")
173216
if result != testutils.SUCCESS:
174217
return result
175218
# Finally, append the actual runtime command to the process
176-
result = bridge.ns_proc_append(proc, self._get_run_cmd())
219+
result = bridge.ns_proc_write(proc, self._get_run_cmd())
177220
if result != testutils.SUCCESS:
178221
return result
179222
# Execute the command queue and close the process, retrieve result

0 commit comments

Comments
 (0)