diff --git a/configure.ac b/configure.ac index 44f9df5ed9..b73ac53237 100644 --- a/configure.ac +++ b/configure.ac @@ -175,6 +175,10 @@ AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264], [Use x264 library (default: no)]), [], [enable_x264=no]) AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes]) +AC_ARG_ENABLE(openh264, AS_HELP_STRING([--enable-openh264], + [Use openh264 library (default: no)]), + [], [enable_openh264=no]) +AM_CONDITIONAL(XRDP_OPENH264, [test x$enable_openh264 = xyes]) AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter], [Do not use included painter library (default: no)]), [], [enable_painter=yes]) @@ -493,6 +497,8 @@ AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 > AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] ) +AS_IF( [test "x$enable_openh264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_OPENH264, openh264 >= 2.0.0)] ) + # checking for TurboJPEG if test "x$enable_tjpeg" = "xyes" then @@ -668,6 +674,7 @@ echo " jpeg $enable_jpeg" echo " turbo jpeg $enable_tjpeg" echo " rfxcodec $enable_rfxcodec" echo " x264 $enable_x264" +echo " openh264 $enable_openh264" echo " painter $enable_painter" echo " pixman $enable_pixman" echo " fuse $enable_fuse" diff --git a/xrdp/Makefile.am b/xrdp/Makefile.am index 4603cf654a..6b9359b9e5 100644 --- a/xrdp/Makefile.am +++ b/xrdp/Makefile.am @@ -33,6 +33,13 @@ XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS) XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h endif +if XRDP_OPENH264 +AM_CPPFLAGS += -DXRDP_OPENH264 +AM_CPPFLAGS += $(XRDP_OPENH264_CFLAGS) +XRDP_EXTRA_LIBS += $(XRDP_OPENH264_LIBS) +XRDP_EXTRA_SOURCES += xrdp_encoder_openh264.c xrdp_encoder_openh264.h +endif + if XRDP_PIXMAN AM_CPPFLAGS += -DXRDP_PIXMAN AM_CPPFLAGS += $(PIXMAN_CFLAGS) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index ed9b4abb09..e53de45716 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -38,6 +38,10 @@ #include "xrdp_encoder_x264.h" #endif +#ifdef XRDP_OPENH264 +#include "xrdp_encoder_openh264.h" +#endif + #define DEFAULT_XRDP_GFX_FRAMES_IN_FLIGHT 2 /* limits used for validate env var XRDP_GFX_FRAMES_IN_FLIGHT */ #define MIN_XRDP_GFX_FRAMES_IN_FLIGHT 1 @@ -94,7 +98,7 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); static int process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); #endif -#ifdef XRDP_X264 +#if defined(XRDP_X264) || defined(XRDP_OPENH264) static int process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); #endif @@ -171,7 +175,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) client_info->capture_format = XRDP_a8b8g8r8; self->process_enc = process_enc_jpg; } -#ifdef XRDP_X264 +#if defined(XRDP_X264) || defined(XRDP_OPENH264) else if (mm->egfx_flags & XRDP_EGFX_H264) { LOG(LOG_LEVEL_INFO, @@ -309,6 +313,56 @@ xrdp_encoder_create(struct xrdp_mm *mm) /* make sure frames_in_flight is at least 1 */ self->frames_in_flight = MAX(self->frames_in_flight, 1); +#if defined(XRDP_X264) && defined(XRDP_OPENH264) + if (self->h264_flags == 0) + { + char *env = g_getenv("XRDP_PREFER_OPENH264"); + if (env != NULL) + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: found env var " + "XRDP_PREFER_OPENH264=%s", env); + if (g_text2bool(env)) + { + ENC_SET_BIT(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT); + } + } + else + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: " + "XRDP_PREFER_OPENH264 not found"); + } + } + if (ENC_IS_BIT_SET(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT)) + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for " + "software encoder"); + self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create; + self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete; + self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode; + } + else + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for " + "software encoder"); + self->xrdp_encoder_h264_create = xrdp_encoder_x264_create; + self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete; + self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode; + } +#elif defined(XRDP_OPENH264) + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for " + "software encoder"); + self->h264_flags |= 1; + self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create; + self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete; + self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode; +#elif defined(XRDP_X264) + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for " + "software encoder"); + self->xrdp_encoder_h264_create = xrdp_encoder_x264_create; + self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete; + self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode; +#endif + /* create thread to process messages */ tc_thread_create(proc_enc_msg, self); @@ -354,17 +408,17 @@ xrdp_encoder_delete(struct xrdp_encoder *self) } #endif -#if defined(XRDP_X264) +#if defined(XRDP_X264) || defined(XRDP_OPENH264) for (index = 0; index < 16; index++) { if (self->codec_handle_h264_gfx[index] != NULL) { - xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]); + self->xrdp_encoder_h264_delete(self->codec_handle_h264_gfx[index]); } } if (self->codec_handle_h264 != NULL) { - xrdp_encoder_x264_delete(self->codec_handle_h264); + self->xrdp_encoder_h264_delete(self->codec_handle_h264); } #endif @@ -628,7 +682,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #endif -#if defined(XRDP_X264) +#if defined(XRDP_X264) || defined(XRDP_OPENH264) /*****************************************************************************/ static int @@ -734,7 +788,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self, struct xrdp_egfx_bulk *bulk, struct stream *in_s, XRDP_ENC_DATA *enc) { -#ifdef XRDP_X264 +#if defined(XRDP_X264) || defined(XRDP_OPENH264) int index; int surface_id; int codec_id; @@ -901,7 +955,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self, if (self->codec_handle_h264_gfx[mon_index] == NULL) { self->codec_handle_h264_gfx[mon_index] = - xrdp_encoder_x264_create(); + self->xrdp_encoder_h264_create(); if (self->codec_handle_h264_gfx[mon_index] == NULL) { g_free(s->data); @@ -909,7 +963,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self, return NULL; } } - error = xrdp_encoder_x264_encode( + error = self->xrdp_encoder_h264_encode( self->codec_handle_h264_gfx[mon_index], 0, 0, 0, width, height, twidth, theight, 0, diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index 3996fac540..b5fcf3f32b 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -14,6 +14,19 @@ struct xrdp_enc_data; +typedef void *(*xrdp_encoder_h264_create_proc)(void); +typedef int (*xrdp_encoder_h264_delete_proc)(void *handle); +typedef int (*xrdp_encoder_h264_encode_proc)( + void *handle, int session, int left, int top, + int width, int height, int twidth, int theight, + int format, const char *data, + short *crects, int num_crects, + char *cdata, int *cdata_bytes, + int connection_type, int *flags_ptr); + +/* h264_flags */ +#define ENC_FLAGS_PREFER_OPENH264_BIT 0 + /* for codec mode operations */ struct xrdp_encoder { @@ -46,6 +59,11 @@ struct xrdp_encoder int quant_idx_y; int quant_idx_u; int quant_idx_v; + int h264_flags; + int pad0; + xrdp_encoder_h264_create_proc xrdp_encoder_h264_create; + xrdp_encoder_h264_delete_proc xrdp_encoder_h264_delete; + xrdp_encoder_h264_encode_proc xrdp_encoder_h264_encode; }; /* cmd_id = 0 */ diff --git a/xrdp/xrdp_encoder_openh264.c b/xrdp/xrdp_encoder_openh264.c new file mode 100644 index 0000000000..f695dd9a2c --- /dev/null +++ b/xrdp/xrdp_encoder_openh264.c @@ -0,0 +1,308 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016-2024 + * Copyright (C) Christopher Pitstick 2023-2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * openh264 Encoder + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "xrdp.h" +#include "arch.h" +#include "os_calls.h" +#include "xrdp_encoder_openh264.h" +#include "xrdp_tconfig.h" + +#define OPENH264_MAX_ENCODERS 16 + +struct openh264_encoder +{ + ISVCEncoder *openh264_enc_han; + char *yuvdata; + int width; + int height; +}; + +struct openh264_global +{ + struct openh264_encoder encoders[OPENH264_MAX_ENCODERS]; + struct xrdp_tconfig_gfx_openh264_param + openh264_param[NUM_CONNECTION_TYPES]; +}; + +/*****************************************************************************/ +void * +xrdp_encoder_openh264_create(void) +{ + struct openh264_global *og; + struct xrdp_tconfig_gfx gfxconfig; + + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_openh264_create:"); + og = g_new0(struct openh264_global, 1); + tconfig_load_gfx(GFX_CONF, &gfxconfig); + memcpy(&og->openh264_param, &gfxconfig.openh264_param, + sizeof(struct xrdp_tconfig_gfx_openh264_param) * + NUM_CONNECTION_TYPES); + return og; + +} + +/*****************************************************************************/ +int +xrdp_encoder_openh264_delete(void *handle) +{ + struct openh264_global *og; + struct openh264_encoder *oe; + int index; + + if (handle == NULL) + { + return 0; + } + og = (struct openh264_global *) handle; + for (index = 0; index < 16; index++) + { + oe = &(og->encoders[index]); + if (oe->openh264_enc_han != NULL) + { + WelsDestroySVCEncoder(oe->openh264_enc_han); + } + g_free(oe->yuvdata); + } + g_free(og); + return 0; +} + +/*****************************************************************************/ +int +xrdp_encoder_openh264_encode(void *handle, int session, int left, int top, + int width, int height, int twidth, int theight, + int format, const char *data, + short *crects, int num_crects, + char *cdata, int *cdata_bytes, + int connection_type, int *flags_ptr) +{ + struct openh264_global *og; + struct openh264_encoder *oe; + const char *src8; + const char *src8a; + char *dst8; + char *dst8a; + char *dst8b; + char *dst8c; + int index; + int jndex; + int flags; + int x; + int y; + int cx; + int cy; + int ct; /* connection_type */ + SSourcePicture pic1; + SFrameBSInfo info; + SLayerBSInfo *layer; + SEncParamExt encParamExt; + SSpatialLayerConfig *slc; + int status; + int layer_position; + char *write_location; + unsigned char *payload; + int size; + int lcdata_bytes; + + LOG(LOG_LEVEL_TRACE, "xrdp_encoder_openh264_encode:"); + flags = 0; + og = (struct openh264_global *) handle; + oe = &(og->encoders[session % OPENH264_MAX_ENCODERS]); + /* validate connection type */ + ct = connection_type; + if (ct > CONNECTION_TYPE_LAN || ct < CONNECTION_TYPE_MODEM) + { + ct = CONNECTION_TYPE_LAN; + } + if ((oe->openh264_enc_han == NULL) || + (oe->width != width) || (oe->height != height)) + { + if (oe->openh264_enc_han != NULL) + { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: " + "WelsDestroySVCEncoder %p", oe->openh264_enc_han); + WelsDestroySVCEncoder(oe->openh264_enc_han); + oe->openh264_enc_han = NULL; + g_free(oe->yuvdata); + oe->yuvdata = NULL; + flags |= 2; + } + if ((width > 0) && (height > 0)) + { + status = WelsCreateSVCEncoder(&oe->openh264_enc_han); + if ((status != 0) || (oe->openh264_enc_han == NULL)) + { + LOG(LOG_LEVEL_ERROR, "Failed to create H.264 encoder"); + return 1; + } + LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: " + "WelsCreateSVCEncoder rv %p for width %d height %d", + oe->openh264_enc_han, width, height); + status = (*oe->openh264_enc_han)->GetDefaultParams( + oe->openh264_enc_han, &encParamExt); + LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: " + "GetDefaultParams rv %d", status); + if (status == 0) + { + encParamExt.iUsageType = CAMERA_VIDEO_REAL_TIME; + encParamExt.iPicWidth = (width + 15) & ~15; + encParamExt.iPicHeight = (height + 15) & ~15; + encParamExt.iRCMode = RC_OFF_MODE; + encParamExt.bEnableFrameSkip = 0; + encParamExt.iSpatialLayerNum = 1; + /* defaults to INCREASING_ID, Mac client needs CONSTANT_ID */ + encParamExt.eSpsPpsIdStrategy = CONSTANT_ID; + slc = encParamExt.sSpatialLayers + 0; + slc->fFrameRate = encParamExt.fMaxFrameRate; + slc->iVideoWidth = encParamExt.iPicWidth; + slc->iVideoHeight = encParamExt.iPicHeight; + slc->iSpatialBitrate = encParamExt.iTargetBitrate; + slc->iMaxSpatialBitrate = encParamExt.iMaxBitrate; + status = (*oe->openh264_enc_han)->InitializeExt( + oe->openh264_enc_han, &encParamExt); + LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: " + "InitializeExt rv %d", status); + } + oe->yuvdata = g_new(char, (width + 16) * (height + 16) * 2); + if (oe->yuvdata == NULL) + { + WelsDestroySVCEncoder(oe->openh264_enc_han); + oe->openh264_enc_han = NULL; + return 2; + } + flags |= 1; + } + oe->width = width; + oe->height = height; + } + + if ((data != NULL) && (oe->openh264_enc_han != NULL)) + { + g_memset(&pic1, 0, sizeof(pic1)); + pic1.iPicWidth = (width + 15) & ~15; + pic1.iPicHeight = (height + 15) & ~15; + pic1.iColorFormat = videoFormatI420; + pic1.iStride[0] = pic1.iPicWidth; + pic1.iStride[1] = pic1.iPicWidth / 2; + pic1.iStride[2] = pic1.iPicWidth / 2; + pic1.pData[0] = (unsigned char *) (oe->yuvdata); + pic1.pData[1] = pic1.pData[0] + pic1.iPicWidth * pic1.iPicHeight; + pic1.pData[2] = pic1.pData[1] + (pic1.iPicWidth / 2) * + (pic1.iPicHeight / 2); + for (index = 0; index < num_crects; index++) + { + src8 = data; + dst8 = (char *) (pic1.pData[0]); + x = crects[index * 4 + 0]; + y = crects[index * 4 + 1]; + cx = crects[index * 4 + 2]; + cy = crects[index * 4 + 3]; + LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: " + "x %d y %d cx %d cy %d", x, y, cx, cy); + src8 += twidth * y + x; + dst8 += pic1.iStride[0] * (y - top) + (x - left); + for (; cy > 0; cy -= 1) + { + g_memcpy(dst8, src8, cx); + src8 += twidth; + dst8 += pic1.iStride[0]; + } + } + for (index = 0; index < num_crects; index++) + { + src8 = data; /* uv */ + src8 += twidth * theight; + dst8 = (char *) (pic1.pData[1]); /* u */ + dst8a = (char *) (pic1.pData[2]); /* v */ + x = crects[index * 4 + 0]; + y = crects[index * 4 + 1]; + cx = crects[index * 4 + 2]; + cy = crects[index * 4 + 3]; + src8 += twidth * (y / 2) + x; + dst8 += pic1.iStride[1] * ((y - top) / 2) + ((x - left) / 2); + dst8a += pic1.iStride[2] * ((y - top) / 2) + ((x - left) / 2); + for (; cy > 0; cy -= 2) + { + src8a = src8; /* uv */ + dst8b = dst8; /* u */ + dst8c = dst8a; /* v */ + for (jndex = 0; jndex < cx; jndex += 2) + { + *(dst8b++) = *(src8a++); /* u */ + *(dst8c++) = *(src8a++); /* v */ + } + src8 += twidth; /* uv */ + dst8 += pic1.iStride[1]; /* u */ + dst8a += pic1.iStride[2]; /* v */ + } + } + g_memset(&info, 0, sizeof(info)); + status = (*oe->openh264_enc_han)->EncodeFrame(oe->openh264_enc_han, + &pic1, &info); + if (status != 0) + { + LOG(LOG_LEVEL_INFO, "Failed to encode frame"); + return 3; + } + if (info.eFrameType == videoFrameTypeSkip) + { + LOG(LOG_LEVEL_INFO, "frame was skipped!"); + return 4; + } + lcdata_bytes = 0; + for (index = 0; index < info.iLayerNum; index++) + { + layer_position = 0; + layer = info.sLayerInfo + index; + for (jndex = 0; jndex < layer->iNalCount; jndex++) + { + write_location = cdata + lcdata_bytes; + payload = layer->pBsBuf + layer_position; + size = layer->pNalLengthInByte[jndex]; + if (lcdata_bytes + size > *cdata_bytes) + { + LOG(LOG_LEVEL_INFO, "out of room"); + return 5; + } + g_memcpy(write_location, payload, size); + layer_position += size; + lcdata_bytes += size; + } + } + *cdata_bytes = lcdata_bytes; + } + if (flags_ptr != NULL) + { + *flags_ptr = flags; + } + return 0; +} diff --git a/xrdp/xrdp_encoder_openh264.h b/xrdp/xrdp_encoder_openh264.h new file mode 100644 index 0000000000..6c40f2e179 --- /dev/null +++ b/xrdp/xrdp_encoder_openh264.h @@ -0,0 +1,39 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016-2024 + * Copyright (C) Christopher Pitstick 2023-2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * openh264 Encoder + */ + +#ifndef _XRDP_ENCODER_OPENH264_H +#define _XRDP_ENCODER_OPENH264_H + +#include "arch.h" + +void * +xrdp_encoder_openh264_create(void); +int +xrdp_encoder_openh264_delete(void *handle); +int +xrdp_encoder_openh264_encode(void *handle, int session, int left, int top, + int width, int height, int twidth, int theight, + int format, const char *data, + short *crects, int num_crects, + char *cdata, int *cdata_bytes, + int connection_type, int *flags_ptr); + +#endif diff --git a/xrdp/xrdp_tconfig.h b/xrdp/xrdp_tconfig.h index 649d649028..2bebee8a6f 100644 --- a/xrdp/xrdp_tconfig.h +++ b/xrdp/xrdp_tconfig.h @@ -42,6 +42,11 @@ struct xrdp_tconfig_gfx_x264_param int fps_den; }; +struct xrdp_tconfig_gfx_openh264_param +{ + int pad0; +}; + enum xrdp_tconfig_codecs { XTC_H264, @@ -59,6 +64,8 @@ struct xrdp_tconfig_gfx struct xrdp_tconfig_gfx_codec_order codec; /* store x264 parameters for each connection type */ struct xrdp_tconfig_gfx_x264_param x264_param[NUM_CONNECTION_TYPES]; + struct xrdp_tconfig_gfx_openh264_param + openh264_param[NUM_CONNECTION_TYPES]; }; static const char *const rdpbcgr_connection_type_names[] =