Skip to content

Commit

Permalink
Check OpenH264 is working before using it
Browse files Browse the repository at this point in the history
  • Loading branch information
@matt335672
matt335672 committed Jan 29, 2025
1 parent 7eb96dd commit e52171d
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 32 deletions.
100 changes: 68 additions & 32 deletions xrdp/xrdp_encoder.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,28 @@ xrdp_enc_data_done_destructor(void *item, void *closure)
g_free(enc_done);
}

/*****************************************************************************/
#if defined(XRDP_OPENH264)
// Also applies if both codecs are enabled
static int
check_h264_ok(void)
{
int rv = xrdp_encoder_openh264_install_ok();
if (!rv)
{
LOG(LOG_LEVEL_ERROR, "OpenH264 Codec is not installed correctly. "
"H.264 will not be used");
}
return rv;
}
#elif defined(XRDP_X264)
static int
check_h264_ok(void)
{
return 1;
}
#endif

/*****************************************************************************/
struct xrdp_encoder *
xrdp_encoder_create(struct xrdp_mm *mm)
Expand Down Expand Up @@ -165,6 +187,11 @@ xrdp_encoder_create(struct xrdp_mm *mm)
}
self->mm = mm;
self->process_enc = process_enc_egfx;

#if defined(XRDP_X264) || defined(XRDP_OPENH264)
int h264_ok = check_h264_ok();
#endif

if (client_info->jpeg_codec_id != 0)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting jpeg codec session");
Expand All @@ -176,7 +203,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
self->process_enc = process_enc_jpg;
}
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
else if (mm->egfx_flags & XRDP_EGFX_H264)
else if (h264_ok && mm->egfx_flags & XRDP_EGFX_H264)
{
LOG(LOG_LEVEL_INFO,
"xrdp_encoder_create: starting h264 codec session gfx");
Expand All @@ -185,7 +212,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
client_info->capture_format = XRDP_nv12_709fr;
self->gfx = 1;
}
else if (client_info->h264_codec_id != 0)
else if (h264_ok && client_info->h264_codec_id != 0)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session");
self->codec_id = client_info->h264_codec_id;
Expand Down Expand Up @@ -314,40 +341,49 @@ xrdp_encoder_create(struct xrdp_mm *mm)
self->frames_in_flight = MAX(self->frames_in_flight, 1);

#if defined(XRDP_X264) && defined(XRDP_OPENH264)
struct xrdp_tconfig_gfx gfxconfig;
tconfig_load_gfx(GFX_CONF, &gfxconfig);

switch (gfxconfig.h264_encoder)
if (h264_ok)
{
case XTC_H264_OPENH264:
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
break;
case XTC_H264_X264:
default:
/* x264 is the default H.264 software encoder */
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
break;
struct xrdp_tconfig_gfx gfxconfig;
tconfig_load_gfx(GFX_CONF, &gfxconfig);

switch (gfxconfig.h264_encoder)
{
case XTC_H264_OPENH264:
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
break;
case XTC_H264_X264:
default:
/* x264 is the default H.264 software encoder */
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
break;
}
}
#elif defined(XRDP_OPENH264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
if (h264_ok)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
}
#elif defined(XRDP_X264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
if (h264_ok)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
}
#endif

/* create thread to process messages */
Expand Down
33 changes: 33 additions & 0 deletions xrdp/xrdp_encoder_openh264.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,3 +323,36 @@ xrdp_encoder_openh264_encode(void *handle, int session, int left, int top,
}
return 0;
}

/*****************************************************************************/
int
xrdp_encoder_openh264_install_ok(void)
{
int rv;

// Declare something we can take the address of to pass to
// WelsCreateSVCEncoder. This object is not directly accessed.
//
// Note we can't use the ISVCEncoder type directly, as in C++ this
// is an abstract class.
struct
{
long double d;
} dummy;

ISVCEncoder *p = (ISVCEncoder *)&dummy;

// The real OpenH264 library will ALWAYS change the value of the
// passed-in pointer
// The noopenh264 library will NEVER change the value of the passed-in
// pointer
// For both libraries, the relevant source is in
// codec/encoder/plus/src/welsEncoderExt.cpp
WelsCreateSVCEncoder(&p);
rv = (p != (ISVCEncoder *)&dummy); // Did the passed-in value change
// If p is &dummy or NULL, this call does nothing, otherwise resources
// are deallocated.
WelsDestroySVCEncoder(p);

return rv;
}
11 changes: 11 additions & 0 deletions xrdp/xrdp_encoder_openh264.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,15 @@ xrdp_encoder_openh264_encode(void *handle, int session, int left, int top,
char *cdata, int *cdata_bytes,
int connection_type, int *flags_ptr);

/**
* Test OpenH264 library is installed correctly
*
* It's possible on some distros to install the noopenh264 package
* (https://gitlab.com/freedesktop-sdk/noopenh264) for building. If this
* is installed at runtime, openh264 cannot be used.
* @return Boolean (!= 0 -> working)
*/
int
xrdp_encoder_openh264_install_ok(void);

#endif

0 comments on commit e52171d

Please sign in to comment.