From 6174d8c0d4a7b1e6aa35eea84973d398e7aa2877 Mon Sep 17 00:00:00 2001 From: Anton Protopopov Date: Fri, 8 Nov 2019 16:55:30 -0500 Subject: [PATCH] bpf: support XDP metadata An XDP program can use the bpf_xdp_adjust_meta() hook to prepend up to 32 bytes of meta data before the packet. Support this functionality by copying XDP Meta to the local memory after a BPF program execution and pushing in back right before delivering the packet. Signed-off-by: Anton Protopopov --- firmware/apps/nic/datapath.uc | 2 +- firmware/apps/nic/ebpf.uc | 13 +++- firmware/apps/nic/pv.uc | 123 +++++++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/firmware/apps/nic/datapath.uc b/firmware/apps/nic/datapath.uc index 0b8a28f2..4931e6b1 100644 --- a/firmware/apps/nic/datapath.uc +++ b/firmware/apps/nic/datapath.uc @@ -41,7 +41,7 @@ actions#: actions_execute(pkt_vec, egress#) ebpf_reentry#: - ebpf_reentry() + ebpf_reentry(pkt_vec) #pragma warning(disable: 4702) fatal_error("MAIN LOOP EXIT") diff --git a/firmware/apps/nic/ebpf.uc b/firmware/apps/nic/ebpf.uc index c5314eec..fd8a9758 100644 --- a/firmware/apps/nic/ebpf.uc +++ b/firmware/apps/nic/ebpf.uc @@ -134,7 +134,7 @@ ebpf_init_cap_finalize() #define _ebpf_pkt_vec *l$index1 -#macro ebpf_reentry() +#macro ebpf_reentry(in_vec) .begin .reg egress_q_base .reg stat @@ -143,6 +143,10 @@ ebpf_init_cap_finalize() .reg_addr ebpf_rc 0 A .set ebpf_rc .reg rc + .reg xdp_meta_len + + // compute XDP meta length and save the meta in local memory + pv_save_xdp_meta(xdp_meta_len, in_vec) pv_restore_meta_lm_ptr(_ebpf_pkt_vec) @@ -160,6 +164,13 @@ ebpf_init_cap_finalize() pv_set_tx_flag(_ebpf_pkt_vec, BF_L(PV_TX_HOST_RX_BPF_bf)) pv_invalidate_cache(_ebpf_pkt_vec) + // store xdp meta length if it is non-zero + alu[--, --, B, xdp_meta_len] + beq[pass#] + pv_meta_prepend(in_vec, xdp_meta_len) + pv_meta_push_type__sz1(in_vec, NFP_NET_META_XDP_META_LEN) +pass#: + __actions_restore_t_idx() br_bset[rc, EBPF_RET_PASS, actions#] diff --git a/firmware/apps/nic/pv.uc b/firmware/apps/nic/pv.uc index 71e5229d..13c77d48 100644 --- a/firmware/apps/nic/pv.uc +++ b/firmware/apps/nic/pv.uc @@ -74,9 +74,9 @@ passert(PV_MAX_CLONES, "EQ", 2) * +-+---+-----+-------------------+-----+-------------------------+ * 10 |A| 0 | Packet Number | 0 | Offset | 2 * +-+---------+-------------------+-----+---------+---------------+ - * 11 | Sequence Number | --- | Seq Ctx | Protocol | 3 + * 11 | Sequence Number | XML | Seq Ctx | Protocol | 3 * +-------------------------------+-+-+-+---------+---+-+-+-+-+-+-+ - * 12 | TX Host Flags |M|B|Seek (64B algn)|-|Q|I|i|C|c| 4 + * 12 | TX Host Flags |M|B|Seek (64B algn)|X|Q|I|i|C|c| 4 * +-------------------------------+-+-+---------------+-+-+-+-+-+-+ * 13 | 8B Header Offsets (stacked outermost to innermost) | 5 * +-----------------+-----+-----------------------+---------------+ @@ -84,13 +84,17 @@ passert(PV_MAX_CLONES, "EQ", 2) * +-----------------+-----+-----------------------+---------------+ * 15 | Metadata Type Fields | 7 * +---------------------------------------------------------------+ + * 16-23 | XDP Metadata | 0-7 <- ACTIVE_LM_ADDR_2 () + * +---------------------------------------------------------------+ * * 0 - Intentionally zero for efficient extraction and manipulation * S - Split packet * A - 1 if CTM buffer is allocated (ie. packet number and CTM address valid) * CBS - CTM Buffer Size * BLS - Buffer List + * XML - XDP Meta Length = (XML + 1) << 2 * P - Packet pending (multicast) + * X - XDP Meta is present * Q - Queue offset selected (overrides RSS) * V - One or more VLANs present * M - dest MAC is multicast @@ -131,7 +135,7 @@ passert(PV_MAX_CLONES, "EQ", 2) * +-+---------------+ */ -#define PV_SIZE_LW 16 +#define PV_SIZE_LW 24 #define PV_LENGTH_wrd 0 #define PV_CTM_ISL_bf PV_LENGTH_wrd, 31, 26 @@ -151,6 +155,7 @@ passert(PV_MAX_CLONES, "EQ", 2) #define PV_SEQ_wrd 3 #define PV_SEQ_NO_bf PV_SEQ_wrd, 31, 16 +#define PV_XDP_META_LEN_bf PV_SEQ_wrd, 15, 13 #define PV_SEQ_CTX_bf PV_SEQ_wrd, 12, 8 #define PV_PROTO_bf PV_SEQ_wrd, 7, 0 #define PV_PROTO_IPV4_bf PV_SEQ_wrd, 1, 1 @@ -178,6 +183,7 @@ passert(PV_MAX_CLONES, "EQ", 2) #define PV_MAC_DST_MC_bf PV_FLAGS_wrd, 15, 15 #define PV_MAC_DST_BC_bf PV_FLAGS_wrd, 14, 14 #define PV_SEEK_BASE_bf PV_FLAGS_wrd, 13, 6 +#define PV_XDP_META_bf PV_FLAGS_wrd, 5, 5 #define PV_QUEUE_SELECTED_bf PV_FLAGS_wrd, 4, 4 #define PV_CSUM_OFFLOAD_bf PV_FLAGS_wrd, 3, 0 #define PV_CSUM_OFFLOAD_IL3_bf PV_FLAGS_wrd, 3, 3 @@ -450,6 +456,7 @@ passert(PV_MAX_CLONES, "EQ", 2) .reg meta_base .reg ref_cnt .reg write $meta[9] + .reg xdp_meta_len .xfer_order $meta .sig sig_meta @@ -464,6 +471,9 @@ passert(PV_MAX_CLONES, "EQ", 2) alu[out_meta_len, --, B, ref_cnt, <<2] alu[lm_ptr, lm_ptr, -, out_meta_len] + pv_restore_xdp_meta(xdp_meta_len, in_vec) + alu[out_meta_len, out_meta_len, +, xdp_meta_len] + alu[idx, 8, -, ref_cnt] jump[idx, t8#], targets[t8#, t7#, t6#, t5#, t4#, t3#, t2#, t1#], defer[3] local_csr_wr[ACTIVE_LM_ADDR_2, lm_ptr] @@ -505,6 +515,113 @@ end#: .end #endm +#macro pv_get_xdp_meta_len(out_len, in_vec, RET_LABEL) +.begin + + .reg xml + + br_bclr[BF_AL(in_vec, PV_XDP_META_bf), RET_LABEL], defer[1] + immed[out_len, 0] + + bitfield_extract__sz1(xml, BF_AML(in_vec, PV_XDP_META_LEN_bf)) + alu[out_len, xml, +, 1] + alu[out_len, --, B, out_len, <<2] + +.end +#endm + +#macro pv_save_xdp_meta(xdp_meta_len, in_vec) +.begin + .reg offset + .reg lm_ptr + .reg in_pkt_addr_hi + .reg in_pkt_addr_lo + .reg xdp_meta_base + .reg read $tr + .sig sig_rd + + pv_get_xdp_meta_len(xdp_meta_len, in_vec, end#) + pv_get_base_addr(in_pkt_addr_hi, in_pkt_addr_lo, in_vec) + alu[xdp_meta_base, in_pkt_addr_lo, -, xdp_meta_len] + + local_csr_rd[ACTIVE_LM_ADDR_1] + immed[lm_ptr, 0] + alu[lm_ptr, lm_ptr, +, 32] + local_csr_wr[ACTIVE_LM_ADDR_2, lm_ptr] + + alu[offset, 32, -, xdp_meta_len] // jump offset + jump[offset, t8#], targets[t8#, t7#, t6#, t5#, t4#, t3#, t2#, t1#] + +#define LOOP 8 +#while (LOOP > 1) + +t/**/LOOP#: + mem[read32, $tr, in_pkt_addr_hi, <<8, xdp_meta_base, 1], ctx_swap[sig_rd] + alu[*l$index2++, --, B, $tr] + alu[xdp_meta_base, xdp_meta_base, +, 4] + nop + +#define_eval LOOP (LOOP - 1) +#endloop +#undef LOOP + +t1#: + mem[read32, $tr, in_pkt_addr_hi, <<8, xdp_meta_base, 1], ctx_swap[sig_rd] + alu[*l$index2++, --, B, $tr] + +end#: + +.end +#endm + +#macro pv_restore_xdp_meta(xdp_meta_len, in_vec) +.begin + + .reg in_pkt_addr_hi + .reg in_pkt_addr_lo + .reg xdp_meta_base + .reg idx + .reg lm_ptr + .reg ref_cnt + .reg write $tr[8] + .xfer_order $tr + .sig sig_wr + + aggregate_directive(.set_wr, $tr, 8) + + pv_get_xdp_meta_len(xdp_meta_len, in_vec, end#) + pv_get_base_addr(in_pkt_addr_hi, in_pkt_addr_lo, in_vec) + alu[xdp_meta_base, in_pkt_addr_lo, -, xdp_meta_len] + + local_csr_rd[ACTIVE_LM_ADDR_1] + immed[lm_ptr, 0] + alu[lm_ptr, lm_ptr, +, (32-4)] + alu[lm_ptr, lm_ptr, +, xdp_meta_len] + local_csr_wr[ACTIVE_LM_ADDR_2, lm_ptr] + + alu[ref_cnt, --, B, xdp_meta_len, >>2] + alu[idx, 8, -, ref_cnt] + jump[idx, t7#], targets[t7#, t6#, t5#, t4#, t3#, t2#, t1#, t0#] + +#define LOOP 7 +#while (LOOP > 0) +t/**/LOOP#: + alu[$tr[LOOP], --, B, *l$index2--] +#define_eval LOOP (LOOP - 1) +#endloop +#undef LOOP +t0#: + + ov_single(OV_LENGTH, ref_cnt, OVF_SUBTRACT_ONE) + #pragma warning(disable:5009) + mem[write32, $tr[0], in_pkt_addr_hi, <<8, xdp_meta_base, max_8], indirect_ref, ctx_swap[sig_wr], defer[1] + alu[$tr[0], --, B, *l$index2--] + #pragma warning(default:5009) + +end#: + +.end +#endm #macro pv_multicast_init(io_vec, in_bls, CONTINUE_LABEL) .begin