Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add openh264 support #3311

Merged
merged 1 commit into from
Nov 29, 2024
Merged

add openh264 support #3311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,10 @@ AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
[Use x264 library (default: no)]),
[], [enable_x264=no])
AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
AC_ARG_ENABLE(openh264, AS_HELP_STRING([--enable-openh264],
[Use openh264 library (default: no)]),
[], [enable_openh264=no])
AM_CONDITIONAL(XRDP_OPENH264, [test x$enable_openh264 = xyes])
AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
[Do not use included painter library (default: no)]),
[], [enable_painter=yes])
Expand Down Expand Up @@ -493,6 +497,8 @@ AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >

AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )

AS_IF( [test "x$enable_openh264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_OPENH264, openh264 >= 2.0.0)] )

# checking for TurboJPEG
if test "x$enable_tjpeg" = "xyes"
then
Expand Down Expand Up @@ -668,6 +674,7 @@ echo " jpeg $enable_jpeg"
echo " turbo jpeg $enable_tjpeg"
echo " rfxcodec $enable_rfxcodec"
echo " x264 $enable_x264"
echo " openh264 $enable_openh264"
echo " painter $enable_painter"
echo " pixman $enable_pixman"
echo " fuse $enable_fuse"
Expand Down
7 changes: 7 additions & 0 deletions xrdp/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS)
XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h
endif

if XRDP_OPENH264
AM_CPPFLAGS += -DXRDP_OPENH264
AM_CPPFLAGS += $(XRDP_OPENH264_CFLAGS)
XRDP_EXTRA_LIBS += $(XRDP_OPENH264_LIBS)
XRDP_EXTRA_SOURCES += xrdp_encoder_openh264.c xrdp_encoder_openh264.h
endif

if XRDP_PIXMAN
AM_CPPFLAGS += -DXRDP_PIXMAN
AM_CPPFLAGS += $(PIXMAN_CFLAGS)
Expand Down
72 changes: 63 additions & 9 deletions xrdp/xrdp_encoder.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
#include "xrdp_encoder_x264.h"
#endif

#ifdef XRDP_OPENH264
#include "xrdp_encoder_openh264.h"
#endif

#define DEFAULT_XRDP_GFX_FRAMES_IN_FLIGHT 2
/* limits used for validate env var XRDP_GFX_FRAMES_IN_FLIGHT */
#define MIN_XRDP_GFX_FRAMES_IN_FLIGHT 1
Expand Down Expand Up @@ -94,7 +98,7 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
static int
process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
#endif
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
static int
process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
#endif
Expand Down Expand Up @@ -171,7 +175,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
client_info->capture_format = XRDP_a8b8g8r8;
self->process_enc = process_enc_jpg;
}
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
else if (mm->egfx_flags & XRDP_EGFX_H264)
{
LOG(LOG_LEVEL_INFO,
Expand Down Expand Up @@ -309,6 +313,56 @@ xrdp_encoder_create(struct xrdp_mm *mm)
/* make sure frames_in_flight is at least 1 */
self->frames_in_flight = MAX(self->frames_in_flight, 1);

#if defined(XRDP_X264) && defined(XRDP_OPENH264)
if (self->h264_flags == 0)
{
char *env = g_getenv("XRDP_PREFER_OPENH264");
if (env != NULL)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: found env var "
"XRDP_PREFER_OPENH264=%s", env);
if (g_text2bool(env))
{
ENC_SET_BIT(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT);
}
}
else
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: "
"XRDP_PREFER_OPENH264 not found");
}
}
if (ENC_IS_BIT_SET(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT))
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
}
else
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
}
#elif defined(XRDP_OPENH264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
"software encoder");
self->h264_flags |= 1;
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
#elif defined(XRDP_X264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
#endif

/* create thread to process messages */
tc_thread_create(proc_enc_msg, self);

Expand Down Expand Up @@ -354,17 +408,17 @@ xrdp_encoder_delete(struct xrdp_encoder *self)
}
#endif

#if defined(XRDP_X264)
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
for (index = 0; index < 16; index++)
{
if (self->codec_handle_h264_gfx[index] != NULL)
{
xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]);
self->xrdp_encoder_h264_delete(self->codec_handle_h264_gfx[index]);
}
}
if (self->codec_handle_h264 != NULL)
{
xrdp_encoder_x264_delete(self->codec_handle_h264);
self->xrdp_encoder_h264_delete(self->codec_handle_h264);
}
#endif

Expand Down Expand Up @@ -628,7 +682,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
}
#endif

#if defined(XRDP_X264)
#if defined(XRDP_X264) || defined(XRDP_OPENH264)

/*****************************************************************************/
static int
Expand Down Expand Up @@ -734,7 +788,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
struct xrdp_egfx_bulk *bulk, struct stream *in_s,
XRDP_ENC_DATA *enc)
{
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
int index;
int surface_id;
int codec_id;
Expand Down Expand Up @@ -901,15 +955,15 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
if (self->codec_handle_h264_gfx[mon_index] == NULL)
{
self->codec_handle_h264_gfx[mon_index] =
xrdp_encoder_x264_create();
self->xrdp_encoder_h264_create();
if (self->codec_handle_h264_gfx[mon_index] == NULL)
{
g_free(s->data);
g_free(crects);
return NULL;
}
}
error = xrdp_encoder_x264_encode(
error = self->xrdp_encoder_h264_encode(
self->codec_handle_h264_gfx[mon_index], 0,
0, 0,
width, height, twidth, theight, 0,
Expand Down
18 changes: 18 additions & 0 deletions xrdp/xrdp_encoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,19 @@

struct xrdp_enc_data;

typedef void *(*xrdp_encoder_h264_create_proc)(void);
typedef int (*xrdp_encoder_h264_delete_proc)(void *handle);
typedef int (*xrdp_encoder_h264_encode_proc)(
void *handle, int session, int left, int top,
int width, int height, int twidth, int theight,
int format, const char *data,
short *crects, int num_crects,
char *cdata, int *cdata_bytes,
int connection_type, int *flags_ptr);

/* h264_flags */
#define ENC_FLAGS_PREFER_OPENH264_BIT 0

/* for codec mode operations */
struct xrdp_encoder
{
Expand Down Expand Up @@ -46,6 +59,11 @@ struct xrdp_encoder
int quant_idx_y;
int quant_idx_u;
int quant_idx_v;
int h264_flags;
int pad0;
xrdp_encoder_h264_create_proc xrdp_encoder_h264_create;
xrdp_encoder_h264_delete_proc xrdp_encoder_h264_delete;
xrdp_encoder_h264_encode_proc xrdp_encoder_h264_encode;
};

/* cmd_id = 0 */
Expand Down
Loading