GFX: add config which to prefer H264 vs RFX

tests/xrdp/Makefile.am

@@ -1,6 +1,5 @@
 AM_CPPFLAGS = \
   -DXRDP_TOP_SRCDIR=\"$(top_srcdir)\" \
-  -DXRDP_CFG_PATH=\"${sysconfdir}/xrdp\" \
   -I$(top_builddir) \
   -I$(top_srcdir)/xrdp \
   -I$(top_srcdir)/libxrdp \
@@ -21,7 +20,14 @@ EXTRA_DIST = \
   test_not4_8bit.bmp \
   test_not4_24bit.bmp \
   test1.jpg \
-  test_alpha_blend.png
+  test_alpha_blend.png \
+  gfx/gfx.toml\
+  gfx/gfx_codec_order_undefined.toml \
+  gfx/gfx_codec_h264_preferred.toml \
+  gfx/gfx_codec_h264_only.toml \
+  gfx/gfx_codec_rfx_preferred.toml \
+  gfx/gfx_codec_rfx_preferred_odd.toml \
+  gfx/gfx_codec_rfx_only.toml
 
 TESTS = test_xrdp
 check_PROGRAMS = test_xrdp

tests/xrdp/gfx/gfx.toml

@@ -0,0 +1,40 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+# inherits default
+
+[x264.wan]
+vbv_max_bitrate = 15000
+vbv_buffer_size = 1500
+
+[x264.broadband_high]
+preset = "superfast"
+vbv_max_bitrate = 8000
+vbv_buffer_Size = 800
+
+[x264.satellite]
+preset = "superfast"
+vbv_max_bitrate = 5000
+vbv_buffer_size = 500
+
+[x264.broadband_low]
+preset = "veryfast"
+tune = "zerolatency"
+vbv_max_bitrate = 1600
+vbv_buffer_size = 66
+
+[x264.modem]
+preset = "fast"
+tune = "zerolatency"
+vbv_max_bitrate = 1200
+vbv_buffer_size = 50

tests/xrdp/gfx/gfx_codec_h264_only.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ "H.264" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_codec_h264_preferred.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_codec_order_undefined.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_codec_rfx_only.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ "RFX" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_codec_rfx_preferred.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ "RFX", "H.264" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_codec_rfx_preferred_odd.toml

@@ -0,0 +1,18 @@
+[codec]
+order = [ "RFX", "H.264", "RFX" ]
+
+[x264.default]
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 24
+fps_den = 1
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/gfx/gfx_missing_h264.toml

@@ -0,0 +1,9 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/test_tconfig.c

@@ -6,6 +6,8 @@
 #include "test_xrdp.h"
 #include "xrdp.h"
 
+#define GFXCONF_STUBDIR XRDP_TOP_SRCDIR "/tests/xrdp/gfx/"
+
 START_TEST(test_tconfig_gfx_always_success)
 {
     ck_assert_int_eq(1, 1);
@@ -15,7 +17,7 @@ END_TEST
 START_TEST(test_tconfig_gfx_x264_load_basic)
 {
     struct xrdp_tconfig_gfx gfxconfig;
-    int rv = tconfig_load_gfx(XRDP_TOP_SRCDIR "/xrdp/gfx.toml", &gfxconfig);
+    int rv = tconfig_load_gfx(GFXCONF_STUBDIR "/gfx.toml", &gfxconfig);
 
     ck_assert_int_eq(rv, 0);
 
@@ -31,6 +33,68 @@ START_TEST(test_tconfig_gfx_x264_load_basic)
 }
 END_TEST
 
+START_TEST(test_tconfig_gfx_codec_order)
+{
+    struct xrdp_tconfig_gfx gfxconfig;
+
+    /* H264 earlier */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_h264_preferred.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 2);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_H264);
+    ck_assert_int_eq(gfxconfig.codec.codecs[1], XTC_RFX);
+
+    /* H264 only */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_h264_only.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_H264);
+
+    /* RFX earlier */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_rfx_preferred.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 2);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+    ck_assert_int_eq(gfxconfig.codec.codecs[1], XTC_H264);
+
+    /* RFX appears twice like: RFX, H264, RFX */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_rfx_preferred_odd.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 2);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+    ck_assert_int_eq(gfxconfig.codec.codecs[1], XTC_H264);
+
+    /* RFX only */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_rfx_only.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+
+    /* H264 is preferred if order undefined */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_codec_order_undefined.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 2);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_H264);
+    ck_assert_int_eq(gfxconfig.codec.codecs[1], XTC_RFX);
+}
+END_TEST
+
+START_TEST(test_tconfig_gfx_missing_file)
+{
+    struct xrdp_tconfig_gfx gfxconfig;
+
+    /* Check RFX config is returned if the file doesn't exist */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/no_such_file.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+}
+END_TEST
+
+START_TEST(test_tconfig_gfx_missing_h264)
+{
+    struct xrdp_tconfig_gfx gfxconfig;
+
+    /* Check RFX config only is returned if H.264 parameters are missing */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_missing_h264.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+}
+END_TEST
+
 /******************************************************************************/
 Suite *
 make_suite_tconfig_load_gfx(void)
@@ -43,6 +107,9 @@ make_suite_tconfig_load_gfx(void)
     tc_tconfig_load_gfx = tcase_create("xrdp_tconfig_load_gfx");
     tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_always_success);
     tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_x264_load_basic);
+    tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_codec_order);
+    tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_missing_file);
+    tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_missing_h264);
 
     suite_add_tcase(s, tc_tconfig_load_gfx);
 

xrdp/gfx.toml

@@ -1,3 +1,6 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
 [x264.default]
 preset = "ultrafast"
 tune = "zerolatency"

xrdp/xrdp.h

@@ -38,6 +38,14 @@
 #include "xrdp_client_info.h"
 #include "log.h"
 
+#if defined(XRDP_X264) || defined(XRDP_OPENH264) || defined(XRDP_NVENC)
+#if !defined(XRDP_H264)
+#define XRDP_H264 1
+#endif
+#else
+#undef XRDP_H264
+#endif
+
 /* xrdp.c */
 long
 g_xrdp_sync(long (*sync_func)(long param1, long param2), long sync_param1,

xrdp/xrdp_mm.c

@@ -1357,14 +1357,17 @@ xrdp_mm_egfx_caps_advertise(void *user, int caps_count,
     struct xrdp_mm *self;
     struct xrdp_bitmap *screen;
     int index;
-    int best_index;
     int best_h264_index;
     int best_pro_index;
     int error;
     int version;
     int flags;
     struct ver_flags_t *ver_flags;
 
+#if !defined(XRDP_H264)
+    UNUSED_VAR(best_h264_index);
+#endif
+
     LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise:");
     self = (struct xrdp_mm *) user;
     screen = self->wm->screen;
@@ -1385,7 +1388,6 @@ xrdp_mm_egfx_caps_advertise(void *user, int caps_count,
     }
     /* sort by version */
     g_qsort(ver_flags, caps_count, sizeof(struct ver_flags_t), cmpverfunc);
-    best_index = -1;
     best_h264_index = -1;
     best_pro_index = -1;
     for (index = 0; index < caps_count; index++)
@@ -1432,19 +1434,34 @@ xrdp_mm_egfx_caps_advertise(void *user, int caps_count,
                 break;
         }
     }
-    if (best_pro_index >= 0)
-    {
-        best_index = best_pro_index;
-        self->egfx_flags = XRDP_EGFX_RFX_PRO;
-    }
-    /* prefer h264, todo use setting in xrdp.ini for this */
-    if (best_h264_index >= 0)
+
+    int best_index = -1;
+    struct xrdp_tconfig_gfx_codec_order *co = &self->wm->gfx_config->codec;
+    char cobuff[64];
+
+    LOG(LOG_LEVEL_INFO, "Codec search order is %s",
+        tconfig_codec_order_to_str(co, cobuff, sizeof(cobuff)));
+    for (index = 0 ; index < co->codec_count ; ++index)
     {
-#if defined(XRDP_X264) || defined(XRDP_NVENC)
-        best_index = best_h264_index;
-        self->egfx_flags = XRDP_EGFX_H264;
+#if defined(XRDP_H264)
+        if (co->codecs[index] == XTC_H264 && best_h264_index >= 0)
+        {
+            LOG(LOG_LEVEL_INFO, "Matched H264 mode");
+            best_index = best_h264_index;
+            self->egfx_flags = XRDP_EGFX_H264;
+            break;
+        }
 #endif
+
+        if (co->codecs[index] == XTC_RFX && best_pro_index >= 0)
+        {
+            LOG(LOG_LEVEL_INFO, "Matched RFX mode");
+            best_index = best_pro_index;
+            self->egfx_flags = XRDP_EGFX_RFX_PRO;
+            break;
+        }
     }
+
     if (best_index >= 0)
     {
         LOG(LOG_LEVEL_INFO, "  replying version 0x%8.8x flags 0x%8.8x",