Skip to content

Commit

Permalink
H264Libav: Use AVFrame for pixel format conversion buffer
Browse files Browse the repository at this point in the history
This ensures that the buffer is allocated with the correct alignment and
padding for use with sws_scale.

This fixes out-of-bounds writes which would in some cases cause
segmentation faults and/or heap corruption.
  • Loading branch information
any1 committed Aug 16, 2024
1 parent 4f6a352 commit fff17ca
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 11 deletions.
27 changes: 17 additions & 10 deletions common/rfb/H264LibavDecoderContext.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ bool H264LibavDecoderContext::initCodec() {
os::AutoMutex lock(&mutex);

sws = nullptr;
swsBuffer = nullptr;
h264WorkBuffer = nullptr;
h264WorkBufferLength = 0;

Expand Down Expand Up @@ -87,9 +86,6 @@ bool H264LibavDecoderContext::initCodec() {
return false;
}

int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB32, rect.width(), rect.height(), 1);
swsBuffer = new uint8_t[numBytes];

initialized = true;
return true;
}
Expand All @@ -101,8 +97,8 @@ void H264LibavDecoderContext::freeCodec() {
return;
av_parser_close(parser);
avcodec_free_context(&avctx);
av_frame_free(&rgbFrame);
av_frame_free(&frame);
delete[] swsBuffer;
free(h264WorkBuffer);
initialized = false;
}
Expand Down Expand Up @@ -220,11 +216,22 @@ void H264LibavDecoderContext::decode(const uint8_t* h264_in_buffer,
frame->width, frame->height, AV_PIX_FMT_RGB32,
0, nullptr, nullptr, nullptr);

int stride;
pb->getBuffer(rect, &stride);
int dst_linesize = stride * pb->getPF().bpp/8; // stride is in pixels, linesize is in bytes (stride x4). We need bytes
if (rgbFrame && (rgbFrame->width != frame->width || rgbFrame->height != frame->height)) {
av_frame_free(&rgbFrame);

}

if (!rgbFrame) {
rgbFrame = av_frame_alloc();
// TODO: Can we really assume that the pixel format will always be RGB32?
rgbFrame->format = AV_PIX_FMT_RGB32;
rgbFrame->width = frame->width;
rgbFrame->height = frame->height;
av_frame_get_buffer(rgbFrame, 0);
}

sws_scale(sws, frame->data, frame->linesize, 0, frame->height, &swsBuffer, &dst_linesize);
sws_scale(sws, frame->data, frame->linesize, 0, frame->height, rgbFrame->data,
rgbFrame->linesize);

pb->imageRect(rect, swsBuffer, stride);
pb->imageRect(rect, rgbFrame->data[0], rgbFrame->linesize[0] / 4);
}
2 changes: 1 addition & 1 deletion common/rfb/H264LibavDecoderContext.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ namespace rfb {
AVCodecContext *avctx;
AVCodecParserContext *parser;
AVFrame* frame;
AVFrame* rgbFrame = nullptr;
SwsContext* sws;
uint8_t* swsBuffer;
uint8_t* h264WorkBuffer;
uint32_t h264WorkBufferLength;
};
Expand Down

0 comments on commit fff17ca

Please sign in to comment.