From e9d657d1e8ed0ebbe262f85ca96caaeab428e7b8 Mon Sep 17 00:00:00 2001 From: Roberto Scolaro Date: Thu, 2 May 2024 08:31:36 +0000 Subject: [PATCH] fix(bpf,modern_bpf): better 32bit support preadv/pwritev family Signed-off-by: Roberto Scolaro --- driver/bpf/fillers.h | 135 ++++++++++++++++++ .../helpers/store/auxmap_store_params.h | 31 +++- .../helpers/store/ringbuf_store_params.h | 36 ++++- 3 files changed, 193 insertions(+), 9 deletions(-) diff --git a/driver/bpf/fillers.h b/driver/bpf/fillers.h index 4c12733a679..eb5b44ef649 100644 --- a/driver/bpf/fillers.h +++ b/driver/bpf/fillers.h @@ -571,6 +571,111 @@ FILLER(sys_poll_x, true) #define MAX_IOVCNT 32 +static __always_inline int bpf_parse_readv_writev_bufs_ia32(struct filler_data *data, + const struct compat_iovec __user *iovsrc, + unsigned long iovcnt, + long retval, + int flags) +{ + const struct compat_iovec *iov; + int res = PPM_SUCCESS; + unsigned int copylen; + long size = 0; + int j; + + copylen = iovcnt * sizeof(struct compat_iovec); + iov = (const struct compat_iovec *)data->tmp_scratch; + + if (copylen > SCRATCH_SIZE_MAX) + { + return PPM_FAILURE_FRAME_SCRATCH_MAP_FULL; + } + +#ifdef BPF_FORBIDS_ZERO_ACCESS + if (copylen) + if (bpf_probe_read_user((void *)iov, + ((copylen - 1) & SCRATCH_SIZE_MAX) + 1, + (void *)iovsrc)) +#else + if (bpf_probe_read_user((void *)iov, + copylen & SCRATCH_SIZE_MAX, + (void *)iovsrc)) +#endif + return PPM_FAILURE_INVALID_USER_MEMORY; + + + #pragma unroll + for (j = 0; j < MAX_IOVCNT; ++j) { + if (j == iovcnt) + break; + // BPF seems to require a hard limit to avoid overflows + if (size == LONG_MAX) + break; + + size += iov[j].iov_len; + } + + if ((flags & PRB_FLAG_IS_WRITE) == 0) + if (size > retval) + size = retval; + + if (flags & PRB_FLAG_PUSH_SIZE) { + res = bpf_push_u32_to_ring(data, (uint32_t)size); + CHECK_RES(res); + } + + if (flags & PRB_FLAG_PUSH_DATA) { + if (size > 0) { + unsigned long off = _READ(data->state->tail_ctx.curoff); + unsigned long remaining = size; + int j; + + #pragma unroll + for (j = 0; j < MAX_IOVCNT; ++j) { + volatile unsigned int to_read; + + if (j == iovcnt) + break; + + unsigned long off_bounded = off & SCRATCH_SIZE_HALF; + if (off > SCRATCH_SIZE_HALF) + break; + + if (iov[j].iov_len <= remaining) + to_read = iov[j].iov_len; + else + to_read = remaining; + + if (to_read > SCRATCH_SIZE_HALF) + to_read = SCRATCH_SIZE_HALF; + +#ifdef BPF_FORBIDS_ZERO_ACCESS + if (to_read) + if (bpf_probe_read_user(&data->buf[off_bounded], + ((to_read - 1) & SCRATCH_SIZE_HALF) + 1, + iov[j].iov_base)) +#else + if (bpf_probe_read_user(&data->buf[off_bounded], + to_read & SCRATCH_SIZE_HALF, + iov[j].iov_base)) +#endif + return PPM_FAILURE_INVALID_USER_MEMORY; + + remaining -= to_read; + off += to_read; + } + } else { + size = 0; + } + + data->fd = bpf_syscall_get_argument(data, 0); + data->curarg_already_on_frame = true; + return __bpf_val_to_ring(data, 0, size, PT_BYTEBUF, -1, true, KERNEL); + } + + return res; +} + static __always_inline int bpf_parse_readv_writev_bufs(struct filler_data *data, const struct iovec __user *iovsrc, unsigned long iovcnt, @@ -789,11 +894,26 @@ FILLER(sys_writev_pwritev_x, true) */ val = bpf_syscall_get_argument(data, 1); iovcnt = bpf_syscall_get_argument(data, 2); +#if defined(CONFIG_X86_64) && defined(CONFIG_IA32_EMULATION) + if (!bpf_in_ia32_syscall()) +#endif + { res = bpf_parse_readv_writev_bufs(data, (const struct iovec __user *)val, iovcnt, 0, PRB_FLAG_PUSH_DATA | PRB_FLAG_IS_WRITE); + } +#if defined(CONFIG_X86_64) && defined(CONFIG_IA32_EMULATION) + else + { + res = bpf_parse_readv_writev_bufs_ia32(data, + (const struct compat_iovec __user *)val, + iovcnt, + 0, + PRB_FLAG_PUSH_DATA | PRB_FLAG_IS_WRITE); + } +#endif /* if there was an error we send an empty param. * we can improve this in the future but at least we don't lose the whole event. @@ -4002,11 +4122,26 @@ FILLER(sys_pwritev_e, true) unsigned long iov_cnt = bpf_syscall_get_argument(data, 2); /* Parameter 2: size (type: PT_UINT32) */ +#if defined(CONFIG_X86_64) && defined(CONFIG_IA32_EMULATION) + if (!bpf_in_ia32_syscall()) +#endif + { res = bpf_parse_readv_writev_bufs(data, (const struct iovec __user *)iov_pointer, iov_cnt, 0, PRB_FLAG_PUSH_SIZE | PRB_FLAG_IS_WRITE); + } +#if defined(CONFIG_X86_64) && defined(CONFIG_IA32_EMULATION) + else + { + res = bpf_parse_readv_writev_bufs_ia32(data, + (const struct compat_iovec __user *)iov_pointer, + iov_cnt, + 0, + PRB_FLAG_PUSH_SIZE | PRB_FLAG_IS_WRITE); + } +#endif /* if there was an error we send a size equal to `0`. * we can improve this in the future but at least we don't lose the whole event. diff --git a/driver/modern_bpf/helpers/store/auxmap_store_params.h b/driver/modern_bpf/helpers/store/auxmap_store_params.h index 21c0955b0fa..8bb6f8a9c1e 100644 --- a/driver/modern_bpf/helpers/store/auxmap_store_params.h +++ b/driver/modern_bpf/helpers/store/auxmap_store_params.h @@ -1010,7 +1010,15 @@ static __always_inline void auxmap__store_iovec_size_param(struct auxiliary_map static __always_inline void auxmap__store_iovec_data_param(struct auxiliary_map *auxmap, unsigned long iov_pointer, unsigned long iov_cnt, unsigned long len_to_read) { /* We use the second part of our auxmap as a scratch space. */ - uint32_t total_iovec_size = iov_cnt * bpf_core_type_size(struct iovec); + uint32_t total_iovec_size = 0; + if(!bpf_in_ia32_syscall()) + { + total_iovec_size = iov_cnt * bpf_core_type_size(struct iovec); + } + else + { + total_iovec_size = iov_cnt * bpf_core_type_size(struct compat_iovec); + } if(bpf_probe_read_user((void *)&auxmap->data[MAX_PARAM_SIZE], SAFE_ACCESS(total_iovec_size), (void *)iov_pointer)) @@ -1023,7 +1031,16 @@ static __always_inline void auxmap__store_iovec_data_param(struct auxiliary_map uint32_t total_size_to_read = 0; /* Pointer to iovec structs */ - const struct iovec *iovec = (const struct iovec *)&auxmap->data[MAX_PARAM_SIZE]; + const struct iovec *iovec; + const struct compat_iovec *compat_iovec; + if(!bpf_in_ia32_syscall()) + { + iovec = (const struct iovec *)&auxmap->data[MAX_PARAM_SIZE]; + } + else + { + compat_iovec = (const struct compat_iovec *)&auxmap->data[MAX_PARAM_SIZE]; + } uint64_t initial_payload_pos = auxmap->payload_pos; for(int j = 0; j < MAX_IOVCNT; j++) { @@ -1041,7 +1058,15 @@ static __always_inline void auxmap__store_iovec_data_param(struct auxiliary_map break; } - uint16_t bytes_read = push__bytebuf(auxmap->data, &auxmap->payload_pos, (unsigned long)iovec[j].iov_base, iovec[j].iov_len, USER); + uint16_t bytes_read; + if(!bpf_in_ia32_syscall()) + { + bytes_read = push__bytebuf(auxmap->data, &auxmap->payload_pos, (unsigned long)iovec[j].iov_base, iovec[j].iov_len, USER); + } + else + { + bytes_read = push__bytebuf(auxmap->data, &auxmap->payload_pos, (unsigned long)compat_iovec[j].iov_base, compat_iovec[j].iov_len, USER); + } if(!bytes_read) { push__param_len(auxmap->data, &auxmap->lengths_pos, total_size_to_read); diff --git a/driver/modern_bpf/helpers/store/ringbuf_store_params.h b/driver/modern_bpf/helpers/store/ringbuf_store_params.h index 60d12761c84..98967c64c75 100644 --- a/driver/modern_bpf/helpers/store/ringbuf_store_params.h +++ b/driver/modern_bpf/helpers/store/ringbuf_store_params.h @@ -308,7 +308,16 @@ static __always_inline void ringbuf__store_iovec_size_param(struct ringbuf_struc return; } - uint32_t total_iovec_size = iov_cnt * bpf_core_type_size(struct iovec); + uint32_t total_iovec_size = 0; + if(!bpf_in_ia32_syscall()) + { + total_iovec_size = iov_cnt * bpf_core_type_size(struct iovec); + } + else + { + total_iovec_size = iov_cnt * bpf_core_type_size(struct compat_iovec); + } + if(bpf_probe_read_user((void *)&auxmap->data[0], SAFE_ACCESS(total_iovec_size), (void *)iov_pointer)) @@ -320,14 +329,29 @@ static __always_inline void ringbuf__store_iovec_size_param(struct ringbuf_struc uint32_t total_size_to_read = 0; /* Pointer to iovec structs */ - const struct iovec *iovec = (const struct iovec *)&auxmap->data[0]; - for(int j = 0; j < MAX_IOVCNT; j++) + if(!bpf_in_ia32_syscall()) + { + const struct iovec *iovec = (const struct iovec *)&auxmap->data[0]; + for(int j = 0; j < MAX_IOVCNT; j++) + { + if(j == iov_cnt) + { + break; + } + total_size_to_read += iovec[j].iov_len; + } + } + else { - if(j == iov_cnt) + const struct compat_iovec *iovec = (const struct compat_iovec *)&auxmap->data[0]; + for(int j = 0; j < MAX_IOVCNT; j++) { - break; + if(j == iov_cnt) + { + break; + } + total_size_to_read += iovec[j].iov_len; } - total_size_to_read += iovec[j].iov_len; } ringbuf__store_u32(ringbuf, total_size_to_read); }