From bf77f1012985abd1e3ff8afeec46944fd659ea9a Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 21 Dec 2020 00:33:54 +0200 Subject: [PATCH 1/8] net/tap: Added TUNSETSTEERINGEBPF code. Additional code that will be used for eBPF setting steering routine. --- net/tap-linux.h | 1 + 1 file changed, 1 insertion(+) diff --git a/net/tap-linux.h b/net/tap-linux.h index 2f36d100fc5d..1d06fe0de63b 100644 --- a/net/tap-linux.h +++ b/net/tap-linux.h @@ -31,6 +31,7 @@ #define TUNSETQUEUE _IOW('T', 217, int) #define TUNSETVNETLE _IOW('T', 220, int) #define TUNSETVNETBE _IOW('T', 222, int) +#define TUNSETSTEERINGEBPF _IOR('T', 224, int) #endif From 1a30cd2359fb9fff71b80666b8f0ee550cbcda13 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 21 Dec 2020 00:36:36 +0200 Subject: [PATCH 2/8] net: Added SetSteeringEBPF method for NetClientState. For now, that method supported only by Linux TAP. Linux TAP uses TUNSETSTEERINGEBPF ioctl. --- include/net/net.h | 2 ++ net/tap-bsd.c | 5 +++++ net/tap-linux.c | 13 +++++++++++++ net/tap-solaris.c | 5 +++++ net/tap-stub.c | 5 +++++ net/tap.c | 9 +++++++++ net/tap_int.h | 1 + 7 files changed, 40 insertions(+) diff --git a/include/net/net.h b/include/net/net.h index 919facaad2fa..d77fbb887888 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -61,6 +61,7 @@ typedef int (SetVnetBE)(NetClientState *, bool); typedef struct SocketReadState SocketReadState; typedef void (SocketReadStateFinalize)(SocketReadState *rs); typedef void (NetAnnounce)(NetClientState *); +typedef bool (SetSteeringEBPF)(NetClientState *, int); typedef struct NetClientInfo { NetClientDriver type; @@ -82,6 +83,7 @@ typedef struct NetClientInfo { SetVnetLE *set_vnet_le; SetVnetBE *set_vnet_be; NetAnnounce *announce; + SetSteeringEBPF *set_steering_ebpf; } NetClientInfo; struct NetClientState { diff --git a/net/tap-bsd.c b/net/tap-bsd.c index 77aaf674b19d..4f64f31e986e 100644 --- a/net/tap-bsd.c +++ b/net/tap-bsd.c @@ -259,3 +259,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap-linux.c b/net/tap-linux.c index b0635e9e32ce..958476974010 100644 --- a/net/tap-linux.c +++ b/net/tap-linux.c @@ -316,3 +316,16 @@ int tap_fd_get_ifname(int fd, char *ifname) pstrcpy(ifname, sizeof(ifr.ifr_name), ifr.ifr_name); return 0; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + if (ioctl(fd, TUNSETSTEERINGEBPF, (void *) &prog_fd) != 0) { + error_report("Issue while setting TUNSETSTEERINGEBPF:" + " %s with fd: %d, prog_fd: %d", + strerror(errno), fd, prog_fd); + + return -1; + } + + return 0; +} diff --git a/net/tap-solaris.c b/net/tap-solaris.c index 0475a58207b1..d85224242b30 100644 --- a/net/tap-solaris.c +++ b/net/tap-solaris.c @@ -255,3 +255,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap-stub.c b/net/tap-stub.c index de525a2e69d4..a0fa25804b6f 100644 --- a/net/tap-stub.c +++ b/net/tap-stub.c @@ -85,3 +85,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap.c b/net/tap.c index b7512853f46c..d0c1356a041f 100644 --- a/net/tap.c +++ b/net/tap.c @@ -337,6 +337,14 @@ static void tap_poll(NetClientState *nc, bool enable) tap_write_poll(s, enable); } +static bool tap_set_steering_ebpf(NetClientState *nc, int prog_fd) +{ + TAPState *s = DO_UPCAST(TAPState, nc, nc); + assert(nc->info->type == NET_CLIENT_DRIVER_TAP); + + return tap_fd_set_steering_ebpf(s->fd, prog_fd) == 0; +} + int tap_get_fd(NetClientState *nc) { TAPState *s = DO_UPCAST(TAPState, nc, nc); @@ -362,6 +370,7 @@ static NetClientInfo net_tap_info = { .set_vnet_hdr_len = tap_set_vnet_hdr_len, .set_vnet_le = tap_set_vnet_le, .set_vnet_be = tap_set_vnet_be, + .set_steering_ebpf = tap_set_steering_ebpf, }; static TAPState *net_tap_fd_init(NetClientState *peer, diff --git a/net/tap_int.h b/net/tap_int.h index 225a49ea4843..547f8a5a28f0 100644 --- a/net/tap_int.h +++ b/net/tap_int.h @@ -44,5 +44,6 @@ int tap_fd_set_vnet_be(int fd, int vnet_is_be); int tap_fd_enable(int fd); int tap_fd_disable(int fd); int tap_fd_get_ifname(int fd, char *ifname); +int tap_fd_set_steering_ebpf(int fd, int prog_fd); #endif /* NET_TAP_INT_H */ From ce02ac29b64a3bc762a5188c49048b1147cd5c88 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 21 Dec 2020 00:46:31 +0200 Subject: [PATCH 3/8] ebpf: Added eBPF RSS program. RSS program and Makefile to build it. The bpftool used to generate '.h' file. The data in that file may be loaded by libbpf. EBPF compilation is not required for building qemu. You can use Makefile if you need to regenerate rss.bpf.skeleton.h. Signed-off-by: Yuri Benditovich --- tools/ebpf/Makefile.ebpf | 33 +++ tools/ebpf/rss.bpf.c | 505 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 538 insertions(+) create mode 100755 tools/ebpf/Makefile.ebpf create mode 100644 tools/ebpf/rss.bpf.c diff --git a/tools/ebpf/Makefile.ebpf b/tools/ebpf/Makefile.ebpf new file mode 100755 index 000000000000..d32d1680b849 --- /dev/null +++ b/tools/ebpf/Makefile.ebpf @@ -0,0 +1,33 @@ +OBJS = rss.bpf.o + +LLC ?= llc +CLANG ?= clang +INC_FLAGS = `$(CLANG) -print-file-name=include` +EXTRA_CFLAGS ?= -O2 -emit-llvm -fno-stack-protector + +ifdef linuxhdrs +LINUXINCLUDE = -I $(linuxhdrs)/arch/x86/include/uapi \ + -I $(linuxhdrs)/arch/x86/include/generated/uapi \ + -I $(linuxhdrs)/arch/x86/include/generated \ + -I $(linuxhdrs)/include/generated/uapi \ + -I $(linuxhdrs)/include/uapi \ + -I $(linuxhdrs)/include \ + -I $(linuxhdrs)/tools/lib +INC_FLAGS += -nostdinc -isystem +endif + +all: $(OBJS) + +.PHONY: clean + +clean: + rm -f $(OBJS) + +$(OBJS): %.o:%.c + $(CLANG) $(INC_FLAGS) \ + -D__KERNEL__ -D__ASM_SYSREG_H \ + -I../include $(LINUXINCLUDE) \ + $(EXTRA_CFLAGS) -c $< -o -| $(LLC) -march=bpf -filetype=obj -o $@ + bpftool gen skeleton rss.bpf.o > rss.bpf.skeleton.h + cp rss.bpf.skeleton.h ../../ebpf/ + diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c new file mode 100644 index 000000000000..eb377247fc3c --- /dev/null +++ b/tools/ebpf/rss.bpf.c @@ -0,0 +1,505 @@ +/* + * eBPF RSS program + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Prepare: + * Requires llvm, clang, bpftool, linux kernel tree + * + * Build rss.bpf.skeleton.h: + * make -f Makefile.ebpf clean all + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#define INDIRECTION_TABLE_SIZE 128 +#define HASH_CALCULATION_BUFFER_SIZE 36 + +struct rss_config_t { + __u8 redirect; + __u8 populate_hash; + __u32 hash_types; + __u16 indirections_len; + __u16 default_queue; +}; + +struct toeplitz_key_data_t { + __u32 leftmost_32_bits; + __u8 next_byte[HASH_CALCULATION_BUFFER_SIZE]; +}; + +struct packet_hash_info_t { + __u8 is_ipv4; + __u8 is_ipv6; + __u8 is_udp; + __u8 is_tcp; + __u8 is_ipv6_ext_src; + __u8 is_ipv6_ext_dst; + + __u16 src_port; + __u16 dst_port; + + union { + struct { + __be32 in_src; + __be32 in_dst; + }; + + struct { + struct in6_addr in6_src; + struct in6_addr in6_dst; + struct in6_addr in6_ext_src; + struct in6_addr in6_ext_dst; + }; + }; +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_configurations = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct rss_config_t), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_toeplitz_key = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct toeplitz_key_data_t), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_indirection_table = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = INDIRECTION_TABLE_SIZE, +}; + +static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written, + const void *ptr, size_t size) { + __builtin_memcpy(&rss_input[*bytes_written], ptr, size); + *bytes_written += size; +} + +static inline +void net_toeplitz_add(__u32 *result, + __u8 *input, + __u32 len + , struct toeplitz_key_data_t *key) { + + __u32 accumulator = *result; + __u32 leftmost_32_bits = key->leftmost_32_bits; + __u32 byte; + + for (byte = 0; byte < HASH_CALCULATION_BUFFER_SIZE; byte++) { + __u8 input_byte = input[byte]; + __u8 key_byte = key->next_byte[byte]; + __u8 bit; + + for (bit = 0; bit < 8; bit++) { + if (input_byte & (1 << 7)) { + accumulator ^= leftmost_32_bits; + } + + leftmost_32_bits = + (leftmost_32_bits << 1) | ((key_byte & (1 << 7)) >> 7); + + input_byte <<= 1; + key_byte <<= 1; + } + } + + *result = accumulator; +} + + +static inline int ip6_extension_header_type(__u8 hdr_type) +{ + switch (hdr_type) { + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: + case IPPROTO_FRAGMENT: + case IPPROTO_ICMPV6: + case IPPROTO_NONE: + case IPPROTO_DSTOPTS: + case IPPROTO_MH: + return 1; + default: + return 0; + } +} +/* + * According to https://www.iana.org/assignments/ipv6-parameters/ipv6-parameters.xhtml + * we suspect that there are would be no more than 11 extensions in IPv6 header, + * also there is 27 TLV options for Destination and Hop-by-hop extensions. + * Need to choose reasonable amount of maximum extensions/options we may check to find + * ext src/dst. + */ +#define IP6_EXTENSIONS_COUNT 11 +#define IP6_OPTIONS_COUNT 30 + +static inline void parse_ipv6_ext(struct __sk_buff *skb, + struct packet_hash_info_t *info, + __u8 *l4_protocol, size_t *l4_offset) +{ + if (!ip6_extension_header_type(*l4_protocol)) { + return; + } + + struct ipv6_opt_hdr ext_hdr = {}; + + for (unsigned int i = 0; i < IP6_EXTENSIONS_COUNT; ++i) { + + bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_hdr, + sizeof(ext_hdr), BPF_HDR_START_NET); + + if (*l4_protocol == IPPROTO_ROUTING) { + struct ipv6_rt_hdr ext_rt = {}; + + bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_rt, + sizeof(ext_rt), BPF_HDR_START_NET); + + if ((ext_rt.type == IPV6_SRCRT_TYPE_2) && + (ext_rt.hdrlen == sizeof(struct in6_addr) / 8) && + (ext_rt.segments_left == 1)) { + + bpf_skb_load_bytes_relative(skb, + *l4_offset + offsetof(struct rt2_hdr, addr), + &info->in6_ext_dst, sizeof(info->in6_ext_dst), + BPF_HDR_START_NET); + + info->is_ipv6_ext_dst = 1; + } + + } else if (*l4_protocol == IPPROTO_DSTOPTS) { + struct ipv6_opt_t { + __u8 type; + __u8 length; + } __attribute__((packed)) opt = {}; + + size_t opt_offset = sizeof(ext_hdr); + + for (unsigned int j = 0; j < IP6_OPTIONS_COUNT; ++j) { + bpf_skb_load_bytes_relative(skb, *l4_offset + opt_offset, + &opt, sizeof(opt), BPF_HDR_START_NET); + + opt_offset += (opt.type == IPV6_TLV_PAD1) ? + 1 : opt.length + sizeof(opt); + + if (opt_offset + 1 >= ext_hdr.hdrlen * 8) { + break; + } + + if (opt.type == IPV6_TLV_HAO) { + bpf_skb_load_bytes_relative(skb, + *l4_offset + opt_offset + offsetof(struct ipv6_destopt_hao, addr), + &info->is_ipv6_ext_src, sizeof(info->is_ipv6_ext_src), + BPF_HDR_START_NET); + + info->is_ipv6_ext_src = 1; + break; + } + } + } + + *l4_protocol = ext_hdr.nexthdr; + *l4_offset += (ext_hdr.hdrlen + 1) * 8; + + if (!ip6_extension_header_type(ext_hdr.nexthdr)) { + return; + } + } +} + +static __be16 parse_eth_type(struct __sk_buff *skb) +{ + unsigned int offset = 12; + __be16 ret = 0; + + bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret), + BPF_HDR_START_MAC); + + switch (__be16_to_cpu(ret)) { + case ETH_P_8021AD: + offset += 4; + case ETH_P_8021Q: + offset += 4; + bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret), + BPF_HDR_START_MAC); + default: + break; + } + + return ret; +} + +static inline void parse_packet(struct __sk_buff *skb, + struct packet_hash_info_t *info) +{ + if (!info || !skb) { + return; + } + + size_t l4_offset = 0; + __u8 l4_protocol = 0; + __u16 l3_protocol = __be16_to_cpu(parse_eth_type(skb)); + + if (l3_protocol == ETH_P_IP) { + info->is_ipv4 = 1; + + struct iphdr ip = {}; + bpf_skb_load_bytes_relative(skb, 0, &ip, sizeof(ip), + BPF_HDR_START_NET); + + info->in_src = ip.saddr; + info->in_dst = ip.daddr; + + l4_protocol = ip.protocol; + l4_offset = ip.ihl * 4; + } else if (l3_protocol == ETH_P_IPV6) { + info->is_ipv6 = 1; + + struct ipv6hdr ip6 = {}; + bpf_skb_load_bytes_relative(skb, 0, &ip6, sizeof(ip6), + BPF_HDR_START_NET); + + info->in6_src = ip6.saddr; + info->in6_dst = ip6.daddr; + + l4_protocol = ip6.nexthdr; + l4_offset = sizeof(ip6); + + parse_ipv6_ext(skb, info, &l4_protocol, &l4_offset); + } + + if (l4_protocol != 0) { + if (l4_protocol == IPPROTO_TCP) { + info->is_tcp = 1; + + struct tcphdr tcp = {}; + bpf_skb_load_bytes_relative(skb, l4_offset, &tcp, sizeof(tcp), + BPF_HDR_START_NET); + + info->src_port = tcp.source; + info->dst_port = tcp.dest; + } else if (l4_protocol == IPPROTO_UDP) { /* TODO: add udplite? */ + info->is_udp = 1; + + struct udphdr udp = {}; + bpf_skb_load_bytes_relative(skb, l4_offset, &udp, sizeof(udp), + BPF_HDR_START_NET); + + info->src_port = udp.source; + info->dst_port = udp.dest; + } + } +} + +static inline __u32 calculate_rss_hash(struct __sk_buff *skb, + struct rss_config_t *config, struct toeplitz_key_data_t *toe) +{ + __u8 rss_input[HASH_CALCULATION_BUFFER_SIZE] = {}; + size_t bytes_written = 0; + __u32 result = 0; + struct packet_hash_info_t packet_info = {}; + + parse_packet(skb, &packet_info); + + if (packet_info.is_ipv4) { + if (packet_info.is_tcp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (packet_info.is_udp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + } + } else if (packet_info.is_ipv6) { + if (packet_info.is_tcp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { + + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (packet_info.is_udp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { + + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + + } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + } + } + + if (bytes_written) { + net_toeplitz_add(&result, rss_input, bytes_written, toe); + } + + return result; +} + +SEC("tun_rss_steering") +int tun_rss_steering_prog(struct __sk_buff *skb) +{ + + struct rss_config_t *config; + struct toeplitz_key_data_t *toe; + + __u32 key = 0; + __u32 hash = 0; + + config = bpf_map_lookup_elem(&tap_rss_map_configurations, &key); + toe = bpf_map_lookup_elem(&tap_rss_map_toeplitz_key, &key); + + if (config && toe) { + if (!config->redirect) { + return config->default_queue; + } + + hash = calculate_rss_hash(skb, config, toe); + if (hash) { + __u32 table_idx = hash % config->indirections_len; + __u16 *queue = 0; + + queue = bpf_map_lookup_elem(&tap_rss_map_indirection_table, + &table_idx); + + if (queue) { + return *queue; + } + } + + return config->default_queue; + } + + return -1; +} + +char _license[] SEC("license") = "GPL v2"; From a001b46ea2da186bfb1aa42980eb294ca5d11f6e Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 14 Jan 2021 12:09:37 +0200 Subject: [PATCH 4/8] ebpf: Added eBPF RSS loader. Added function that loads RSS eBPF program. Added stub functions for RSS eBPF loader. Added meson and configuration options. By default, eBPF feature enabled if libbpf is present in the build system. libbpf checked in configuration shell script and meson script. Signed-off-by: Yuri Benditovich --- configure | 33 ++++ ebpf/ebpf_rss-stub.c | 40 ++++ ebpf/ebpf_rss.c | 165 +++++++++++++++++ ebpf/ebpf_rss.h | 44 +++++ ebpf/meson.build | 1 + ebpf/rss.bpf.skeleton.h | 397 ++++++++++++++++++++++++++++++++++++++++ ebpf/trace-events | 4 + ebpf/trace.h | 2 + meson.build | 13 ++ 9 files changed, 699 insertions(+) create mode 100644 ebpf/ebpf_rss-stub.c create mode 100644 ebpf/ebpf_rss.c create mode 100644 ebpf/ebpf_rss.h create mode 100644 ebpf/meson.build create mode 100644 ebpf/rss.bpf.skeleton.h create mode 100644 ebpf/trace-events create mode 100644 ebpf/trace.h diff --git a/configure b/configure index 5860bdb77ba8..9d18e941f599 100755 --- a/configure +++ b/configure @@ -342,6 +342,7 @@ vhost_vsock="$default_feature" vhost_user="no" vhost_user_blk_server="auto" vhost_user_fs="$default_feature" +bpf="" kvm="auto" hax="auto" hvf="auto" @@ -1236,6 +1237,10 @@ for opt do ;; --enable-membarrier) membarrier="yes" ;; + --disable-bpf) bpf="no" + ;; + --enable-bpf) bpf="yes" + ;; --disable-blobs) blobs="false" ;; --with-pkgversion=*) pkgversion="$optarg" @@ -1845,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available vhost-user vhost-user backend support vhost-user-blk-server vhost-user-blk server support vhost-vdpa vhost-vdpa kernel backend support + bpf BPF kernel support spice spice rbd rados block device (rbd) libiscsi iscsi support @@ -5057,6 +5063,30 @@ else membarrier=no fi +########################################## +# check for usable bpf system call +if test "$bpf" = ""; then + have_bpf=no + if test "$linux" = "yes" -a "$bigendian" != "yes"; then + cat > $TMPC << EOF + #include + #include + int main(void) { + struct bpf_object *obj = NULL; + bpf_object__load(obj); + exit(0); + } +EOF + if compile_prog "" "-lbpf" ; then + have_bpf=yes + bpf=yes + fi + fi + if test "$have_bpf" = "no"; then + feature_not_found "bpf" "the libbpf is not available" + fi +fi + ########################################## # check if rtnetlink.h exists and is useful have_rtnetlink=no @@ -5905,6 +5935,9 @@ fi if test "$membarrier" = "yes" ; then echo "CONFIG_MEMBARRIER=y" >> $config_host_mak fi +if test "$bpf" = "yes" -a "$bigendian" != "yes" -a "$linux" = "yes" ; then + echo "CONFIG_EBPF=y" >> $config_host_mak +fi if test "$signalfd" = "yes" ; then echo "CONFIG_SIGNALFD=y" >> $config_host_mak fi diff --git a/ebpf/ebpf_rss-stub.c b/ebpf/ebpf_rss-stub.c new file mode 100644 index 000000000000..e71e229190de --- /dev/null +++ b/ebpf/ebpf_rss-stub.c @@ -0,0 +1,40 @@ +/* + * eBPF RSS stub file + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "ebpf/ebpf_rss.h" + +void ebpf_rss_init(struct EBPFRSSContext *ctx) +{ + +} + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) +{ + return false; +} + +bool ebpf_rss_load(struct EBPFRSSContext *ctx) +{ + return false; +} + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key) +{ + return false; +} + +void ebpf_rss_unload(struct EBPFRSSContext *ctx) +{ + +} diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c new file mode 100644 index 000000000000..f7f7102da523 --- /dev/null +++ b/ebpf/ebpf_rss.c @@ -0,0 +1,165 @@ +/* + * eBPF RSS loader + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/error-report.h" + +#include +#include + +#include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */ + +#include "ebpf/ebpf_rss.h" +#include "ebpf/rss.bpf.skeleton.h" +#include "trace.h" + +void ebpf_rss_init(struct EBPFRSSContext *ctx) +{ + if (ctx != NULL) { + ctx->obj = NULL; + } +} + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) +{ + return ctx != NULL && ctx->obj != NULL; +} + +bool ebpf_rss_load(struct EBPFRSSContext *ctx) +{ + struct rss_bpf *rss_bpf_ctx; + + if (ctx == NULL) { + return false; + } + + rss_bpf_ctx = rss_bpf__open(); + if (rss_bpf_ctx == NULL) { + trace_ebpf_error("eBPF RSS", "can not open eBPF RSS object"); + goto l_issue; + } + + bpf_program__set_socket_filter(rss_bpf_ctx->progs.tun_rss_steering_prog); + + if (rss_bpf__load(rss_bpf_ctx)) { + trace_ebpf_error("eBPF RSS", "can not load RSS program"); + goto l_issue; + } + + ctx->obj = rss_bpf_ctx; + ctx->program_fd = bpf_program__fd( + rss_bpf_ctx->progs.tun_rss_steering_prog); + ctx->map_configuration = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_configurations); + ctx->map_indirections_table = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_indirection_table); + ctx->map_toeplitz_key = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_toeplitz_key); + + return true; +l_issue: + rss_bpf__destroy(rss_bpf_ctx); + ctx->obj = NULL; + + return false; +} + +static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx, + struct EBPFRSSConfig *config) +{ + uint32_t map_key = 0; + + if (!ebpf_rss_is_loaded(ctx)) { + return false; + } + if (bpf_map_update_elem(ctx->map_configuration, + &map_key, config, 0) < 0) { + return false; + } + return true; +} + +static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx, + uint16_t *indirections_table, + size_t len) +{ + uint32_t i = 0; + + if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL || + len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { + return false; + } + + for (; i < len; ++i) { + if (bpf_map_update_elem(ctx->map_indirections_table, &i, + indirections_table + i, 0) < 0) { + return false; + } + } + return true; +} + +static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx, + uint8_t *toeplitz_key) +{ + uint32_t map_key = 0; + + /* prepare toeplitz key */ + uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {}; + + if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) { + return false; + } + memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE); + *(uint32_t *)toe = ntohl(*(uint32_t *)toe); + + if (bpf_map_update_elem(ctx->map_toeplitz_key, &map_key, toe, + 0) < 0) { + return false; + } + return true; +} + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key) +{ + if (!ebpf_rss_is_loaded(ctx) || config == NULL || + indirections_table == NULL || toeplitz_key == NULL) { + return false; + } + + if (!ebpf_rss_set_config(ctx, config)) { + return false; + } + + if (!ebpf_rss_set_indirections_table(ctx, indirections_table, + config->indirections_len)) { + return false; + } + + if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) { + return false; + } + + return true; +} + +void ebpf_rss_unload(struct EBPFRSSContext *ctx) +{ + if (!ebpf_rss_is_loaded(ctx)) { + return; + } + + rss_bpf__destroy(ctx->obj); + ctx->obj = NULL; +} diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h new file mode 100644 index 000000000000..0fcb4e010fe3 --- /dev/null +++ b/ebpf/ebpf_rss.h @@ -0,0 +1,44 @@ +/* + * eBPF RSS header + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#ifndef QEMU_EBPF_RSS_H +#define QEMU_EBPF_RSS_H + +struct EBPFRSSContext { + void *obj; + int program_fd; + int map_configuration; + int map_toeplitz_key; + int map_indirections_table; +}; + +struct EBPFRSSConfig { + uint8_t redirect; + uint8_t populate_hash; + uint32_t hash_types; + uint16_t indirections_len; + uint16_t default_queue; +}; + +void ebpf_rss_init(struct EBPFRSSContext *ctx); + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx); + +bool ebpf_rss_load(struct EBPFRSSContext *ctx); + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key); + +void ebpf_rss_unload(struct EBPFRSSContext *ctx); + +#endif /* QEMU_EBPF_RSS_H */ diff --git a/ebpf/meson.build b/ebpf/meson.build new file mode 100644 index 000000000000..f5bd5a0f01d5 --- /dev/null +++ b/ebpf/meson.build @@ -0,0 +1 @@ +common_ss.add(when: 'CONFIG_EBPF', if_true: files('ebpf_rss.c'), if_false: files('ebpf_rss-stub.c')) diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h new file mode 100644 index 000000000000..76616a5ca601 --- /dev/null +++ b/ebpf/rss.bpf.skeleton.h @@ -0,0 +1,397 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +/* THIS FILE IS AUTOGENERATED! */ +#ifndef __RSS_BPF_SKEL_H__ +#define __RSS_BPF_SKEL_H__ + +#include +#include + +struct rss_bpf { + struct bpf_object_skeleton *skeleton; + struct bpf_object *obj; + struct { + struct bpf_map *tap_rss_map_configurations; + struct bpf_map *tap_rss_map_indirection_table; + struct bpf_map *tap_rss_map_toeplitz_key; + } maps; + struct { + struct bpf_program *tun_rss_steering_prog; + } progs; + struct { + struct bpf_link *tun_rss_steering_prog; + } links; +}; + +static void +rss_bpf__destroy(struct rss_bpf *obj) +{ + if (!obj) + return; + if (obj->skeleton) + bpf_object__destroy_skeleton(obj->skeleton); + free(obj); +} + +static inline int +rss_bpf__create_skeleton(struct rss_bpf *obj); + +static inline struct rss_bpf * +rss_bpf__open_opts(const struct bpf_object_open_opts *opts) +{ + struct rss_bpf *obj; + + obj = (struct rss_bpf *)calloc(1, sizeof(*obj)); + if (!obj) + return NULL; + if (rss_bpf__create_skeleton(obj)) + goto err; + if (bpf_object__open_skeleton(obj->skeleton, opts)) + goto err; + + return obj; +err: + rss_bpf__destroy(obj); + return NULL; +} + +static inline struct rss_bpf * +rss_bpf__open(void) +{ + return rss_bpf__open_opts(NULL); +} + +static inline int +rss_bpf__load(struct rss_bpf *obj) +{ + return bpf_object__load_skeleton(obj->skeleton); +} + +static inline struct rss_bpf * +rss_bpf__open_and_load(void) +{ + struct rss_bpf *obj; + + obj = rss_bpf__open(); + if (!obj) + return NULL; + if (rss_bpf__load(obj)) { + rss_bpf__destroy(obj); + return NULL; + } + return obj; +} + +static inline int +rss_bpf__attach(struct rss_bpf *obj) +{ + return bpf_object__attach_skeleton(obj->skeleton); +} + +static inline void +rss_bpf__detach(struct rss_bpf *obj) +{ + return bpf_object__detach_skeleton(obj->skeleton); +} + +static inline int +rss_bpf__create_skeleton(struct rss_bpf *obj) +{ + struct bpf_object_skeleton *s; + + s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s)); + if (!s) + return -1; + obj->skeleton = s; + + s->sz = sizeof(*s); + s->name = "rss_bpf"; + s->obj = &obj->obj; + + /* maps */ + s->map_cnt = 3; + s->map_skel_sz = sizeof(*s->maps); + s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz); + if (!s->maps) + goto err; + + s->maps[0].name = "tap_rss_map_configurations"; + s->maps[0].map = &obj->maps.tap_rss_map_configurations; + + s->maps[1].name = "tap_rss_map_indirection_table"; + s->maps[1].map = &obj->maps.tap_rss_map_indirection_table; + + s->maps[2].name = "tap_rss_map_toeplitz_key"; + s->maps[2].map = &obj->maps.tap_rss_map_toeplitz_key; + + /* programs */ + s->prog_cnt = 1; + s->prog_skel_sz = sizeof(*s->progs); + s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz); + if (!s->progs) + goto err; + + s->progs[0].name = "tun_rss_steering_prog"; + s->progs[0].prog = &obj->progs.tun_rss_steering_prog; + s->progs[0].link = &obj->links.tun_rss_steering_prog; + + s->data_sz = 7088; + s->data = (void *)"\ +\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\x30\x19\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0a\0\ +\x01\0\x7b\x1a\x38\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\ +\xa6\0\0\0\0\0\0\x07\x06\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\xbf\x62\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x18\x01\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\xbf\x62\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\ +\0\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x07\x0f\x02\0\0\0\0\xbf\x89\0\ +\0\0\0\0\0\x15\x09\x0d\x02\0\0\0\0\x71\x71\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\ +\x05\0\x09\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\ +\xff\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\ +\0\0\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\ +\x7b\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\ +\x1a\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\ +\x50\xff\0\0\0\0\x79\xa6\x38\xff\0\0\0\0\x15\x06\x85\0\0\0\0\0\x6b\x1a\xd0\xff\ +\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\ +\xb7\x02\0\0\x0c\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\ +\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\xff\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\ +\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\x55\x03\x08\0\xa8\x88\0\0\xb7\x02\0\ +\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\ +\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\ +\xff\0\0\0\0\xdc\x01\0\0\x10\0\0\0\x15\x01\x26\0\xdd\x86\0\0\x55\x01\x6c\0\0\ +\x08\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\ +\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\ +\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\ +\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x61\xa1\xdc\ +\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x60\xff\0\ +\0\0\0\x71\xa6\xd9\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\ +\x57\x01\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\x57\x06\0\0\xff\0\0\0\x15\x06\ +\x45\0\x11\0\0\0\x79\xa1\x38\xff\0\0\0\0\x55\x06\x52\0\x06\0\0\0\xb7\x02\0\0\ +\x01\0\0\0\x73\x2a\x53\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\x63\x2a\xe0\xff\0\0\0\0\ +\x7b\x2a\xd8\xff\0\0\0\0\x7b\x2a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\ +\0\xd0\xff\xff\xff\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x14\0\0\0\x05\0\x41\0\0\ +\0\0\0\x7b\x7a\x10\xff\0\0\0\0\xb7\x07\0\0\x01\0\0\0\x73\x7a\x51\xff\0\0\0\0\ +\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\ +\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\ +\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\ +\0\xbf\x61\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\x05\0\0\ +\x01\0\0\0\x85\0\0\0\x44\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\ +\x77\x01\0\0\x20\0\0\0\x63\x1a\x60\xff\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\ +\x64\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xe8\xff\ +\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\ +\x79\xa1\xf0\xff\0\0\0\0\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\ +\x78\xff\0\0\0\0\x71\xa6\xd6\xff\0\0\0\0\x25\x06\xb1\0\x3c\0\0\0\x6f\x67\0\0\0\ +\0\0\0\x18\x01\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x17\0\0\0\0\0\0\x55\x07\ +\x01\0\0\0\0\0\x05\0\xab\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xfe\xff\0\0\0\0\ +\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\ +\x8c\xff\xff\xff\x7b\x1a\x20\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x54\ +\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\xb7\x07\0\0\0\0\0\0\x7b\x8a\x30\xff\0\0\0\ +\0\x7b\x9a\x28\xff\0\0\0\0\x05\0\xdf\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\ +\x52\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\ +\0\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\ +\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\ +\xff\0\0\0\0\x6b\x1a\x56\xff\0\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x58\xff\0\ +\0\0\0\x71\xa1\x50\xff\0\0\0\0\x15\x01\x0f\0\0\0\0\0\x61\x71\x04\0\0\0\0\0\x71\ +\xa2\x53\xff\0\0\0\0\x15\x02\x41\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x02\ +\0\0\0\x15\x02\x3e\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\ +\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x69\xa1\x56\xff\0\0\0\0\x6b\ +\x1a\xa8\xff\0\0\0\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xaa\xff\0\0\0\0\x05\0\xf4\ +\0\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x15\x01\x5f\x01\0\0\0\0\x61\x71\x04\0\0\0\0\ +\0\x71\xa2\x53\xff\0\0\0\0\x15\x02\x3d\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\ +\0\x10\0\0\0\x15\x02\x3a\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\ +\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\ +\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x38\ +\0\0\0\x65\x01\x01\0\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\ +\x07\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\ +\x05\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x65\x01\x01\ +\0\xff\xff\xff\xff\xbf\x43\0\0\0\0\0\0\x61\x21\x04\0\0\0\0\0\x67\x01\0\0\x20\0\ +\0\0\x61\x24\0\0\0\0\0\0\x4f\x41\0\0\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x61\x21\ +\x08\0\0\0\0\0\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\ +\x7b\x2a\xa8\xff\0\0\0\0\x61\x31\0\0\0\0\0\0\x61\x32\x04\0\0\0\0\0\x61\x34\x08\ +\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x58\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\ +\0\x69\xa5\x56\xff\0\0\0\0\x6b\x5a\xc0\xff\0\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\ +\x43\0\0\0\0\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\ +\0\0\x7b\x2a\xb0\xff\0\0\0\0\x05\0\xbf\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\ +\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x04\0\0\0\x15\x02\x01\0\0\0\ +\0\0\x05\0\xbc\xff\0\0\0\0\x57\x01\0\0\x01\0\0\0\x15\x01\x24\x01\0\0\0\0\x61\ +\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\ +\xa4\xff\0\0\0\0\x05\0\xb2\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x16\0\0\0\ +\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x13\0\0\0\0\0\xbf\xa2\0\ +\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\ +\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\ +\x01\0\0\0\x01\0\0\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\ +\0\x07\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\ +\x05\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x15\x01\xc3\ +\xff\0\0\0\0\x05\0\xc1\xff\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\ +\x15\x02\x04\x01\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\ +\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\ +\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\x40\0\0\0\x15\x01\x01\0\0\0\0\0\ +\xbf\x32\0\0\0\0\0\0\x61\x23\x04\0\0\0\0\0\x67\x03\0\0\x20\0\0\0\x61\x24\0\0\0\ +\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xa0\xff\0\0\0\0\x61\x23\x08\0\0\0\0\0\x61\ +\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x32\0\0\0\0\0\0\x7b\x2a\xa8\xff\0\ +\0\0\0\x15\x01\x78\0\0\0\0\0\x71\xa1\x55\xff\0\0\0\0\x15\x01\x76\0\0\0\0\0\x61\ +\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\ +\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\ +\x61\xa2\x8c\xff\0\0\0\0\x05\0\x75\0\0\0\0\0\x15\x06\x54\xff\x87\0\0\0\x05\0\ +\x3e\0\0\0\0\0\x0f\x96\0\0\0\0\0\0\xbf\x62\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\ +\x71\xa3\xff\xff\0\0\0\0\x67\x03\0\0\x03\0\0\0\x3d\x32\x22\0\0\0\0\0\x55\x01\ +\x0c\0\xc9\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\x16\0\0\0\0\0\0\x07\x06\0\0\x02\0\ +\0\0\x79\xa1\x38\xff\0\0\0\0\xbf\x62\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\ +\x04\0\0\x01\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xb7\x01\0\0\x01\0\ +\0\0\x73\x1a\x54\xff\0\0\0\0\x05\0\x15\0\0\0\0\0\x07\x08\0\0\xff\xff\xff\xff\ +\xbf\x81\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\xbf\x69\0\0\0\ +\0\0\0\x15\x01\x0f\0\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\ +\x12\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xb7\x06\0\0\ +\x01\0\0\0\x79\xa1\x38\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\ +\x85\0\0\0\x44\0\0\0\x71\xa1\xf8\xff\0\0\0\0\x15\x01\xdb\xff\0\0\0\0\x71\xa6\ +\xf9\xff\0\0\0\0\x07\x06\0\0\x02\0\0\0\x05\0\xd8\xff\0\0\0\0\x79\xa8\x30\xff\0\ +\0\0\0\x79\xa9\x28\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\x67\x01\0\0\x03\0\0\0\ +\x79\xa2\x40\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\x08\0\0\0\x7b\x2a\x40\ +\xff\0\0\0\0\x71\xa6\xfe\xff\0\0\0\0\x25\x06\x37\0\x3c\0\0\0\xb7\x01\0\0\x01\0\ +\0\0\x6f\x61\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\ +\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x30\0\0\0\0\0\x07\x07\0\0\x01\0\0\0\xbf\ +\x71\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x02\0\x0b\ +\0\0\0\x79\xa7\x10\xff\0\0\0\0\x05\0\xda\xfe\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\ +\x03\0\0\xfe\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\ +\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x61\0\0\0\0\0\ +\0\x15\x01\x1b\0\x3c\0\0\0\x55\x01\xe0\xff\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\ +\x1a\xf8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa6\ +\x38\xff\0\0\0\0\xbf\x61\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x04\0\ +\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\ +\xd4\xff\x02\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x55\x01\xd2\xff\x02\0\0\0\x71\xa1\ +\xfb\xff\0\0\0\0\x55\x01\xd0\xff\x01\0\0\0\x79\xa2\x40\xff\0\0\0\0\x07\x02\0\0\ +\x08\0\0\0\xbf\x61\0\0\0\0\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\ +\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\ +\xff\0\0\0\0\x05\0\xc6\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xf8\xff\0\0\0\0\ +\xb7\x09\0\0\x02\0\0\0\xb7\x08\0\0\x1e\0\0\0\x05\0\xb0\xff\0\0\0\0\x15\x06\xcf\ +\xff\x87\0\0\0\x05\0\xd3\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\ +\0\0\x61\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\ +\xa1\x70\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\ +\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x07\x08\0\0\x04\0\0\0\x61\ +\x92\0\0\0\0\0\0\xb7\x05\0\0\0\0\0\0\x05\0\x4e\0\0\0\0\0\xaf\x53\0\0\0\0\0\0\ +\xbf\x85\0\0\0\0\0\0\x0f\x15\0\0\0\0\0\0\x71\x55\0\0\0\0\0\0\x67\x02\0\0\x01\0\ +\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\ +\0\0\0\x67\0\0\0\x39\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\ +\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x06\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\ +\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3a\0\0\0\xc7\0\ +\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\x67\x02\0\0\x01\0\0\0\ +\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x05\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\ +\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3b\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\ +\0\0\xaf\x03\0\0\0\0\0\0\x67\x02\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\ +\x04\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\ +\0\x3c\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\xbf\ +\x50\0\0\0\0\0\0\x77\0\0\0\x03\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\0\x01\0\0\0\ +\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3d\0\0\0\xc7\0\0\0\x3f\0\0\ +\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x02\0\ +\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\ +\0\0\0\0\x67\0\0\0\x3e\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\ +\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x01\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\ +\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\x57\x04\0\0\x01\0\0\0\x87\x04\0\0\0\0\0\0\ +\x5f\x24\0\0\0\0\0\0\xaf\x43\0\0\0\0\0\0\x57\x05\0\0\x01\0\0\0\x67\x02\0\0\x01\ +\0\0\0\x4f\x52\0\0\0\0\0\0\x07\x01\0\0\x01\0\0\0\xbf\x35\0\0\0\0\0\0\x15\x01\ +\x0b\0\x24\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xa0\xff\xff\xff\x0f\x13\0\0\0\ +\0\0\0\x71\x34\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x38\0\0\0\xc7\0\0\0\ +\x38\0\0\0\xb7\x03\0\0\0\0\0\0\x65\0\xa9\xff\xff\xff\xff\xff\xbf\x23\0\0\0\0\0\ +\0\x05\0\xa7\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\ +\x20\0\0\0\x15\x01\x0b\0\0\0\0\0\x69\x72\x08\0\0\0\0\0\x3f\x21\0\0\0\0\0\0\x2f\ +\x21\0\0\0\0\0\0\x1f\x13\0\0\0\0\0\0\x63\x3a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\ +\0\x07\x02\0\0\x50\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\ +\x01\0\0\0\x55\0\x02\0\0\0\0\0\x69\x70\x0a\0\0\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\ +\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\x02\0\0\0\x04\0\0\0\x0c\0\0\0\x01\0\0\0\0\0\0\ +\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\x01\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\ +\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\x20\x76\x32\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x01\ +\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x20\ +\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\0\ +\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd7\x01\0\0\0\0\x03\0\x38\ +\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbf\x01\0\0\0\0\x03\0\xe0\x09\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x68\x01\0\0\0\0\x03\0\x88\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x01\ +\0\0\0\0\x03\0\xf0\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcf\x01\0\0\0\0\x03\0\x10\ +\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x01\0\0\0\0\x03\0\x88\x0a\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x9f\x01\0\0\0\0\x03\0\xb8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\ +\0\0\0\0\x03\0\xf0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf1\0\0\0\0\0\x03\0\x30\x0b\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\x02\0\0\0\0\x03\0\x40\x0b\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\xf7\x01\0\0\0\0\x03\0\x38\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaf\x01\0\0\ +\0\0\x03\0\xb8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x97\x01\0\0\0\0\x03\0\xe0\x0b\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x01\0\0\0\0\x03\0\xc0\x02\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x10\x01\0\0\0\0\x03\0\xf8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe9\0\0\0\0\0\ +\x03\0\x40\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x70\x01\0\0\0\0\x03\0\xb8\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x0f\x02\0\0\0\0\x03\0\x70\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\xa7\x01\0\0\0\0\x03\0\xb0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x8f\x01\0\0\0\0\x03\ +\0\xa0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x28\x01\0\0\0\0\x03\0\xd0\x07\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\xe1\0\0\0\0\0\x03\0\xf8\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x87\ +\x01\0\0\0\0\x03\0\x70\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x01\0\0\0\0\x03\0\ +\x90\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\0\0\x03\0\xc8\x06\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xd9\0\0\0\0\0\x03\0\xd0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x07\ +\x02\0\0\0\0\x03\0\xd8\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xef\x01\0\0\0\0\x03\0\ +\x08\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x58\x01\0\0\0\0\x03\0\x68\x08\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x20\x01\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd1\ +\0\0\0\0\0\x03\0\xb8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe7\x01\0\0\0\0\x03\0\xc8\ +\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7f\x01\0\0\0\0\x03\0\x18\x09\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x30\x01\0\0\0\0\x03\0\x30\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc9\0\0\ +\0\0\0\x03\0\x48\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf9\0\0\0\0\0\x03\0\xc0\x01\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\xff\x01\0\0\0\0\x03\0\x88\x0d\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\xdf\x01\0\0\0\0\x03\0\x98\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc7\x01\0\0\0\ +\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x77\x01\0\0\0\0\x03\0\xc0\x0d\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\x50\x01\0\0\0\0\x03\0\x88\x10\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\x18\x01\0\0\0\0\x03\0\0\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\x03\ +\0\x08\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x10\x11\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6b\0\0\0\x11\0\x06\0\0\0\0\0\0\ +\0\0\0\x07\0\0\0\0\0\0\0\x25\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\ +\0\x82\0\0\0\x11\0\x05\0\x28\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x01\0\0\0\x11\0\ +\x05\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x40\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\ +\0\x20\x11\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x31\0\0\0\x50\0\0\0\0\0\0\0\ +\x01\0\0\0\x33\0\0\0\xe0\x10\0\0\0\0\0\0\x01\0\0\0\x32\0\0\0\x1c\0\0\0\0\0\0\0\ +\x01\0\0\0\x2f\0\0\0\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\x6f\ +\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\x6d\x61\x70\ +\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\ +\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\ +\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\ +\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\ +\0\x2e\x72\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x74\x61\x70\x5f\x72\ +\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\ +\x74\x61\x62\x6c\x65\0\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x2e\x73\x74\x72\ +\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x30\x5f\x39\0\x4c\x42\ +\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x37\x39\0\x4c\x42\x42\x30\x5f\x36\ +\x39\0\x4c\x42\x42\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x34\x39\0\x4c\x42\x42\ +\x30\x5f\x33\x39\0\x4c\x42\x42\x30\x5f\x32\x39\0\x4c\x42\x42\x30\x5f\x38\0\x4c\ +\x42\x42\x30\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x35\x38\0\x4c\x42\x42\x30\x5f\ +\x33\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\x4c\x42\x42\x30\x5f\x36\x37\0\x4c\x42\ +\x42\x30\x5f\x34\x37\0\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x35\ +\x36\0\x4c\x42\x42\x30\x5f\x32\x36\0\x4c\x42\x42\x30\x5f\x31\x36\0\x4c\x42\x42\ +\x30\x5f\x38\x35\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\ +\x4c\x42\x42\x30\x5f\x31\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\ +\x38\x34\0\x4c\x42\x42\x30\x5f\x37\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\ +\x42\x30\x5f\x34\x34\0\x4c\x42\x42\x30\x5f\x33\x34\0\x4c\x42\x42\x30\x5f\x32\ +\x34\0\x4c\x42\x42\x30\x5f\x34\x33\0\x4c\x42\x42\x30\x5f\x33\x33\0\x4c\x42\x42\ +\x30\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x33\0\x4c\x42\x42\x30\x5f\x38\x32\0\ +\x4c\x42\x42\x30\x5f\x32\x32\0\x4c\x42\x42\x30\x5f\x31\x32\0\x4c\x42\x42\x30\ +\x5f\x38\x31\0\x4c\x42\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\ +\x42\x42\x30\x5f\x33\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\x5f\ +\x36\x30\0\x4c\x42\x42\x30\x5f\x34\x30\0\x4c\x42\x42\x30\x5f\x33\x30\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x10\x17\0\0\0\0\0\0\x1f\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x5a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\ +\0\0\0\x20\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\x56\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x16\0\0\0\0\0\0\x30\0\ +\0\0\0\0\0\0\x09\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x20\0\0\0\ +\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x11\0\0\0\0\0\0\x3c\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6c\0\0\0\x01\0\0\0\x03\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9c\x11\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xa8\x11\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x17\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x09\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\ +\0\0\0\0\0\0\0\xb2\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd8\x11\0\0\ +\0\0\0\0\xf8\x04\0\0\0\0\0\0\x01\0\0\0\x30\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\ +\0\0\0"; + + return 0; +err: + bpf_object__destroy_skeleton(s); + return -1; +} + +#endif /* __RSS_BPF_SKEL_H__ */ diff --git a/ebpf/trace-events b/ebpf/trace-events new file mode 100644 index 000000000000..411b1e2be391 --- /dev/null +++ b/ebpf/trace-events @@ -0,0 +1,4 @@ +# See docs/devel/tracing.txt for syntax documentation. + +# ebpf-rss.c +ebpf_error(const char *s1, const char *s2) "error in %s: %s" diff --git a/ebpf/trace.h b/ebpf/trace.h new file mode 100644 index 000000000000..ad570e669119 --- /dev/null +++ b/ebpf/trace.h @@ -0,0 +1,2 @@ +#include "trace/trace-ebpf.h" + diff --git a/meson.build b/meson.build index 563688d682c7..27e41690e7a5 100644 --- a/meson.build +++ b/meson.build @@ -968,6 +968,13 @@ if not get_option('fuse_lseek').disabled() endif endif +# libbpf +libbpf = not_found +if 'CONFIG_EBPF' in config_host + libbpf = cc.find_library('bpf', required: true) + +endif + if get_option('cfi') cfi_flags=[] # Check for dependency on LTO @@ -1685,6 +1692,7 @@ if have_system 'backends', 'backends/tpm', 'chardev', + 'ebpf', 'hw/9pfs', 'hw/acpi', 'hw/alpha', @@ -1850,6 +1858,9 @@ subdir('accel') subdir('plugins') subdir('bsd-user') subdir('linux-user') +subdir('ebpf') + +common_ss.add(libbpf) bsd_user_ss.add(files('gdbstub.c')) specific_ss.add_all(when: 'CONFIG_BSD_USER', if_true: bsd_user_ss) @@ -2415,6 +2426,7 @@ summary_info += {'vhost-user support': config_host.has_key('CONFIG_VHOST_USER')} summary_info += {'vhost-user-blk server support': have_vhost_user_blk_server} summary_info += {'vhost-user-fs support': config_host.has_key('CONFIG_VHOST_USER_FS')} summary_info += {'vhost-vdpa support': config_host.has_key('CONFIG_VHOST_VDPA')} +summary_info += {'bpf support': config_host.has_key('CONFIG_EBPF')} summary_info += {'Trace backends': config_host['TRACE_BACKENDS']} if config_host['TRACE_BACKENDS'].split().contains('simple') summary_info += {'Trace output file': config_host['CONFIG_TRACE_FILE'] + '-'} @@ -2516,3 +2528,4 @@ if not supported_oses.contains(targetos) message('if you care about QEMU on this platform you should contact') message('us upstream at qemu-devel@nongnu.org.') endif + From d3b22462a718181db796f10409298c4436f08ddb Mon Sep 17 00:00:00 2001 From: Yuri Benditovich Date: Sun, 24 Jan 2021 09:49:22 +0200 Subject: [PATCH 5/8] net: add ability to hide (disable) vhost_net if 'vhost_net_disabled' in the NetClientState of the net device, get_vhost_net for TAP returns NULL. Network adapters can use this ability to hide the vhost_net temporary between resets in case some active features contradict with vhost. Signed-off-by: Yuri Benditovich --- hw/net/vhost_net.c | 4 +++- include/net/net.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index 24d555e764a8..6660efd9ea96 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -436,7 +436,9 @@ VHostNetState *get_vhost_net(NetClientState *nc) switch (nc->info->type) { case NET_CLIENT_DRIVER_TAP: - vhost_net = tap_get_vhost_net(nc); + if (!nc->vhost_net_disabled) { + vhost_net = tap_get_vhost_net(nc); + } break; #ifdef CONFIG_VHOST_NET_USER case NET_CLIENT_DRIVER_VHOST_USER: diff --git a/include/net/net.h b/include/net/net.h index d77fbb887888..4829803dc22b 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -102,6 +102,7 @@ struct NetClientState { int vring_enable; int vnet_hdr_len; bool is_netdev; + bool vhost_net_disabled; QTAILQ_HEAD(, NetFilterState) filters; }; From 4d0508a36e356c25137c0d630fd943fe8c05bac4 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 14 Jan 2021 12:11:08 +0200 Subject: [PATCH 6/8] virtio-net: Added eBPF RSS to virtio-net. When RSS is enabled the device tries to load the eBPF program to select RX virtqueue in the TUN. If eBPF can be loaded the RSS will function also with vhost (works with kernel 5.8 and later). Software RSS is used as a fallback with vhost=off when eBPF can't be loaded or when hash population requested by the guest. In case the RSS feature is present and acked by the guest but the ebpf due to any reason can't be used with vhost, the virtio-net disables vhost till next reset (or migration) and uses userspace virtio. The same in case the hash report feature is acked by the guest (the vhost can't support this feature currently). Signed-off-by: Yuri Benditovich --- hw/net/vhost_net.c | 2 + hw/net/virtio-net.c | 156 ++++++++++++++++++++++++++++++--- include/hw/virtio/virtio-net.h | 4 + net/vhost-vdpa.c | 2 + 4 files changed, 153 insertions(+), 11 deletions(-) diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index 6660efd9ea96..23520349a2d7 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -71,6 +71,8 @@ static const int user_feature_bits[] = { VIRTIO_NET_F_MTU, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_RING_PACKED, + VIRTIO_NET_F_RSS, + VIRTIO_NET_F_HASH_REPORT, /* This bit implies RARP isn't sent by QEMU out of band */ VIRTIO_NET_F_GUEST_ANNOUNCE, diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index 09ceb02c9d93..fd9c39fefef9 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -515,6 +515,15 @@ static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) return info; } +static void virtio_net_allow_vhost(VirtIONet *n, bool allow) +{ + int i; + for (i = 0; i < n->max_queues; i++) { + NetClientState *nc = qemu_get_subqueue(n->nic, i)->peer; + nc->vhost_net_disabled = !allow; + } +} + static void virtio_net_reset(VirtIODevice *vdev) { VirtIONet *n = VIRTIO_NET(vdev); @@ -552,6 +561,7 @@ static void virtio_net_reset(VirtIODevice *vdev) assert(!virtio_net_get_subqueue(nc)->async_tx.elem); } } + virtio_net_allow_vhost(n, true); } static void peer_test_vnet_hdr(VirtIONet *n) @@ -596,6 +606,7 @@ static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, n->guest_hdr_len = n->mergeable_rx_bufs ? sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr); + n->rss_data.populate_hash = false; } for (i = 0; i < n->max_queues; i++) { @@ -725,9 +736,8 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, return features; } - virtio_clear_feature(&features, VIRTIO_NET_F_RSS); - virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); features = vhost_net_get_features(get_vhost_net(nc->peer), features); + vdev->backend_features = features; if (n->mtu_bypass_backend && @@ -877,6 +887,8 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) VirtIONet *n = VIRTIO_NET(vdev); Error *err = NULL; int i; + NetClientState *nc = qemu_get_queue(n->nic); + bool disable_vhost = false; if (n->mtu_bypass_backend && !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { @@ -901,6 +913,22 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); + /* + * disable vhost till next reset if there are acked features that + * we can't support with vhost + */ + disable_vhost = n->rss_data.redirect && get_vhost_net(nc->peer) && + !ebpf_rss_is_loaded(&n->ebpf_rss); + + if (n->rss_data.populate_hash && get_vhost_net(nc->peer)) { + disable_vhost = true; + } + + if (disable_vhost) { + warn_report("Not using vhost till next reset"); + virtio_net_allow_vhost(n, false); + } + if (n->has_vnet_hdr) { n->curr_guest_offloads = virtio_net_guest_offloads_by_features(features); @@ -908,7 +936,7 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) } for (i = 0; i < n->max_queues; i++) { - NetClientState *nc = qemu_get_subqueue(n->nic, i); + nc = qemu_get_subqueue(n->nic, i); if (!get_vhost_net(nc->peer)) { continue; @@ -1151,12 +1179,79 @@ static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, } } +static void virtio_net_detach_epbf_rss(VirtIONet *n); + static void virtio_net_disable_rss(VirtIONet *n) { if (n->rss_data.enabled) { trace_virtio_net_rss_disable(); } n->rss_data.enabled = false; + + virtio_net_detach_epbf_rss(n); +} + +static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) +{ + NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); + if (nc == NULL || nc->info->set_steering_ebpf == NULL) { + return false; + } + + return nc->info->set_steering_ebpf(nc, prog_fd); +} + +static void rss_data_to_rss_config(struct VirtioNetRssData *data, + struct EBPFRSSConfig *config) +{ + config->redirect = data->redirect; + config->populate_hash = data->populate_hash; + config->hash_types = data->hash_types; + config->indirections_len = data->indirections_len; + config->default_queue = data->default_queue; +} + +static bool virtio_net_attach_epbf_rss(VirtIONet *n) +{ + struct EBPFRSSConfig config = {}; + + if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { + return false; + } + + rss_data_to_rss_config(&n->rss_data, &config); + + if (!ebpf_rss_set_all(&n->ebpf_rss, &config, + n->rss_data.indirections_table, n->rss_data.key)) { + return false; + } + + if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { + return false; + } + + return true; +} + +static void virtio_net_detach_epbf_rss(VirtIONet *n) +{ + virtio_net_attach_ebpf_to_backend(n->nic, -1); +} + +static bool virtio_net_load_ebpf(VirtIONet *n) +{ + if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { + /* backend does't support steering ebpf */ + return false; + } + + return ebpf_rss_load(&n->ebpf_rss); +} + +static void virtio_net_unload_ebpf(VirtIONet *n) +{ + virtio_net_attach_ebpf_to_backend(n->nic, -1); + ebpf_rss_unload(&n->ebpf_rss); } static uint16_t virtio_net_handle_rss(VirtIONet *n, @@ -1271,6 +1366,25 @@ static uint16_t virtio_net_handle_rss(VirtIONet *n, goto error; } n->rss_data.enabled = true; + + if (!n->rss_data.populate_hash) { + if (!virtio_net_attach_epbf_rss(n)) { + /* EBPF must be loaded for vhost */ + if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { + warn_report("Can't load eBPF RSS for vhost"); + goto error; + } + /* fallback to software RSS */ + warn_report("Can't load eBPF RSS - fallback to software RSS"); + n->rss_data.enabled_software_rss = true; + } + } else { + /* use software RSS for hash populating */ + /* and detach eBPF if was loaded before */ + virtio_net_detach_epbf_rss(n); + n->rss_data.enabled_software_rss = true; + } + trace_virtio_net_rss_enable(n->rss_data.hash_types, n->rss_data.indirections_len, temp.b); @@ -1656,7 +1770,7 @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, return -1; } - if (!no_rss && n->rss_data.enabled) { + if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { int index = virtio_net_process_rss(nc, buf, size); if (index >= 0) { NetClientState *nc2 = qemu_get_subqueue(n->nic, index); @@ -2759,13 +2873,6 @@ static int virtio_net_post_load_device(void *opaque, int version_id) } } - if (n->rss_data.enabled) { - trace_virtio_net_rss_enable(n->rss_data.hash_types, - n->rss_data.indirections_len, - sizeof(n->rss_data.key)); - } else { - trace_virtio_net_rss_disable(); - } return 0; } @@ -2782,6 +2889,26 @@ static int virtio_net_post_load_virtio(VirtIODevice *vdev) virtio_net_apply_guest_offloads(n); } + if (n->rss_data.enabled) { + n->rss_data.enabled_software_rss = n->rss_data.populate_hash; + if (!n->rss_data.populate_hash) { + if (!virtio_net_attach_epbf_rss(n)) { + if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { + error_report("Can't post-load eBPF RSS for vhost"); + } else { + warn_report("Can't post-load eBPF RSS - fallback to software RSS"); + n->rss_data.enabled_software_rss = true; + } + } + } + + trace_virtio_net_rss_enable(n->rss_data.hash_types, + n->rss_data.indirections_len, + sizeof(n->rss_data.key)); + } else { + trace_virtio_net_rss_disable(); + } + return 0; } @@ -3336,6 +3463,9 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp) n->qdev = dev; net_rx_pkt_init(&n->rx_pkt, false); + if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { + virtio_net_load_ebpf(n); + } } static void virtio_net_device_unrealize(DeviceState *dev) @@ -3344,6 +3474,8 @@ static void virtio_net_device_unrealize(DeviceState *dev) VirtIONet *n = VIRTIO_NET(dev); int i, max_queues; + virtio_net_unload_ebpf(n); + /* This will stop vhost backend if appropriate. */ virtio_net_set_status(vdev, 0); @@ -3386,6 +3518,8 @@ static void virtio_net_instance_init(Object *obj) device_add_bootindex_property(obj, &n->nic_conf.bootindex, "bootindex", "/ethernet-phy@0", DEVICE(n)); + + ebpf_rss_init(&n->ebpf_rss); } static int virtio_net_pre_save(void *opaque) diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h index 7e96d193aa60..824a69c23f06 100644 --- a/include/hw/virtio/virtio-net.h +++ b/include/hw/virtio/virtio-net.h @@ -21,6 +21,8 @@ #include "qemu/option_int.h" #include "qom/object.h" +#include "ebpf/ebpf_rss.h" + #define TYPE_VIRTIO_NET "virtio-net-device" OBJECT_DECLARE_SIMPLE_TYPE(VirtIONet, VIRTIO_NET) @@ -130,6 +132,7 @@ typedef struct VirtioNetRscChain { typedef struct VirtioNetRssData { bool enabled; + bool enabled_software_rss; bool redirect; bool populate_hash; uint32_t hash_types; @@ -209,6 +212,7 @@ struct VirtIONet { Notifier migration_state; VirtioNetRssData rss_data; struct NetRxPkt *rx_pkt; + struct EBPFRSSContext ebpf_rss; }; void virtio_net_set_netclient_name(VirtIONet *n, const char *name, diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index fe659ec9e242..8b142155494a 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -54,6 +54,8 @@ const int vdpa_feature_bits[] = { VIRTIO_NET_F_MTU, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_RING_PACKED, + VIRTIO_NET_F_RSS, + VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_STATUS, VHOST_INVALID_FEATURE_BIT From fde431271c51de15e7c1d6a389f1d738647904ea Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 14 Jan 2021 12:12:37 +0200 Subject: [PATCH 7/8] docs: Added eBPF documentation. Also, added maintainers information. Signed-off-by: Yuri Benditovich --- MAINTAINERS | 8 +++ docs/ebpf_rss.rst | 125 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 docs/ebpf_rss.rst diff --git a/MAINTAINERS b/MAINTAINERS index 4be087b88e23..4aafdfce2c9e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3173,6 +3173,14 @@ S: Maintained F: hw/semihosting/ F: include/hw/semihosting/ +EBPF: +M: Jason Wang +R: Andrew Melnychenko +R: Yuri Benditovich +S: Maintained +F: ebpf/* +F: tools/ebpf/* + Build and test automation ------------------------- Build and test automation diff --git a/docs/ebpf_rss.rst b/docs/ebpf_rss.rst new file mode 100644 index 000000000000..5beecfe33f03 --- /dev/null +++ b/docs/ebpf_rss.rst @@ -0,0 +1,125 @@ +=========================== +eBPF RSS virtio-net support +=========================== + +RSS(Receive Side Scaling) is used to distribute network packets to guest virtqueues +by calculating packet hash. Usually every queue is processed then by a specific guest CPU core. + +For now there are 2 RSS implementations in qemu: +- 'in-qemu' RSS (functions if qemu receives network packets, i.e. vhost=off) +- eBPF RSS (can function with also with vhost=on) + +eBPF support (CONFIG_EBPF) is enabled by 'configure' script. +To enable eBPF RSS support use './configure --enable-bpf'. + +If steering BPF is not set for kernel's TUN module, the TUN uses automatic selection +of rx virtqueue based on lookup table built according to calculated symmetric hash +of transmitted packets. +If steering BPF is set for TUN the BPF code calculates the hash of packet header and +returns the virtqueue number to place the packet to. + +Simplified decision formula: + +.. code:: C + + queue_index = indirection_table[hash()%] + + +Not for all packets, the hash can/should be calculated. + +Note: currently, eBPF RSS does not support hash reporting. + +eBPF RSS turned on by different combinations of vhost-net, vitrio-net and tap configurations: + +- eBPF is used: + + tap,vhost=off & virtio-net-pci,rss=on,hash=off + +- eBPF is used: + + tap,vhost=on & virtio-net-pci,rss=on,hash=off + +- 'in-qemu' RSS is used: + + tap,vhost=off & virtio-net-pci,rss=on,hash=on + +- eBPF is used, hash population feature is not reported to the guest: + + tap,vhost=on & virtio-net-pci,rss=on,hash=on + +If CONFIG_EBPF is not set then only 'in-qemu' RSS is supported. +Also 'in-qemu' RSS, as a fallback, is used if the eBPF program failed to load or set to TUN. + +RSS eBPF program +---------------- + +RSS program located in ebpf/rss.bpf.skeleton.h generated by bpftool. +So the program is part of the qemu binary. +Initially, the eBPF program was compiled by clang and source code located at tools/ebpf/rss.bpf.c. +Prerequisites to recompile the eBPF program (regenerate ebpf/rss.bpf.skeleton.h): + + llvm, clang, kernel source tree, bpftool + Adjust 'linuxhdrs' in Makefile.ebpf to reflect the location of the kernel source tree + + $ cd ebpf + $ make -f Makefile.ebpf + +Current eBPF RSS implementation uses 'bounded loops' with 'backward jump instructions' which present in the last kernels. +Overall eBPF RSS works on kernels 5.8+. + +eBPF RSS implementation +----------------------- + +eBPF RSS loading functionality located in ebpf/ebpf_rss.c and ebpf/ebpf_rss.h. + +The `struct EBPFRSSContext` structure that holds 4 file descriptors: + +- ctx - pointer of the libbpf context. +- program_fd - file descriptor of the eBPF RSS program. +- map_configuration - file descriptor of the 'configuration' map. This map contains one element of 'struct EBPFRSSConfig'. This configuration determines eBPF program behavior. +- map_toeplitz_key - file descriptor of the 'Toeplitz key' map. One element of the 40byte key prepared for the hashing algorithm. +- map_indirections_table - 128 elements of queue indexes. + +`struct EBPFRSSConfig` fields: + +- redirect - "boolean" value, should the hash be calculated, on false - `default_queue` would be used as the final decision. +- populate_hash - for now, not used. eBPF RSS doesn't support hash reporting. +- hash_types - binary mask of different hash types. See `VIRTIO_NET_RSS_HASH_TYPE_*` defines. If for packet hash should not be calculated - `default_queue` would be used. +- indirections_len - length of the indirections table, maximum 128. +- default_queue - the queue index that used for packet that shouldn't be hashed. For some packets, the hash can't be calculated(g.e ARP). + +Functions: + +- `ebpf_rss_init()` - sets ctx to NULL, which indicates that EBPFRSSContext is not loaded. +- `ebpf_rss_load()` - creates 3 maps and loads eBPF program from tun_rss_steering.h. Returns 'true' on success. After that, program_fd can be used to set steering for TAP. +- `ebpf_rss_set_all()` - sets values for eBPF maps. `indirections_table` length is in EBPFRSSConfig. `toeplitz_key` is VIRTIO_NET_RSS_MAX_KEY_SIZE aka 40 bytes array. +- `ebpf_rss_unload()` - close all file descriptors and set ctx to NULL. + +Simplified eBPF RSS workflow: + +.. code:: C + + struct EBPFRSSConfig config; + config.redirect = 1; + config.hash_types = VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4; + config.indirections_len = VIRTIO_NET_RSS_MAX_TABLE_LEN; + config.default_queue = 0; + + uint16_t table[VIRTIO_NET_RSS_MAX_TABLE_LEN] = {...}; + uint8_t key[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {...}; + + struct EBPFRSSContext ctx; + ebpf_rss_init(&ctx); + ebpf_rss_load(&ctx); + ebpf_rss_set_all(&ctx, &config, table, key); + if (net_client->info->set_steering_ebpf != NULL) { + net_client->info->set_steering_ebpf(net_client, ctx->program_fd); + } + ... + ebpf_unload(&ctx); + + +NetClientState SetSteeringEBPF() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For now, `set_steering_ebpf()` method supported by Linux TAP NetClientState. The method requires an eBPF program file descriptor as an argument. From 8cbb42610cbc7579eddc531f23350758de9e4aae Mon Sep 17 00:00:00 2001 From: Yuri Benditovich Date: Sun, 24 Jan 2021 09:58:42 +0200 Subject: [PATCH 8/8] not for delivery: testing additions for virtio-net Signed-off-by: Yuri Benditovich --- hw/net/virtio-net.c | 14 +++++++++++--- include/hw/virtio/virtio-net.h | 2 ++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index fd9c39fefef9..2b5cbf1db5b9 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -529,6 +529,9 @@ static void virtio_net_reset(VirtIODevice *vdev) VirtIONet *n = VIRTIO_NET(vdev); int i; + printf("rx received %lu\n", n->debug_rx_received); + n->debug_rx_received = 0; + /* Reset back to compatibility mode */ n->promisc = 1; n->allmulti = 0; @@ -1240,8 +1243,9 @@ static void virtio_net_detach_epbf_rss(VirtIONet *n) static bool virtio_net_load_ebpf(VirtIONet *n) { - if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { + if (!n->allow_ebpf_load || !virtio_net_attach_ebpf_to_backend(n->nic, -1)) { /* backend does't support steering ebpf */ + printf("ebpf is not loaded!\n"); return false; } @@ -2477,11 +2481,14 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t size) { VirtIONet *n = qemu_get_nic_opaque(nc); + ssize_t res; if ((n->rsc4_enabled || n->rsc6_enabled)) { - return virtio_net_rsc_receive(nc, buf, size); + res = virtio_net_rsc_receive(nc, buf, size); } else { - return virtio_net_do_receive(nc, buf, size); + res = virtio_net_do_receive(nc, buf, size); } + n->debug_rx_received += res; + return res; } static int32_t virtio_net_flush_tx(VirtIONetQueue *q); @@ -3631,6 +3638,7 @@ static Property virtio_net_properties[] = { DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), + DEFINE_PROP_BOOL("use-ebpf", VirtIONet, allow_ebpf_load, true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h index 824a69c23f06..963e5603cae0 100644 --- a/include/hw/virtio/virtio-net.h +++ b/include/hw/virtio/virtio-net.h @@ -182,6 +182,8 @@ struct VirtIONet { uint8_t nouni; uint8_t nobcast; uint8_t vhost_started; + size_t debug_rx_received; + bool allow_ebpf_load; struct { uint32_t in_use; uint32_t first_multi;