improve full range handling, both in and out

- Fixes a bug in avfilter ignores consumer.color_range
- Removes forcing full range 10-bit to use mlt_image_rgb

As a result of this and the previous change 56a4588 now one can use
mlt_image_format=yuvp10 pix_fmt=yuv410p10le on the consumer and without
Movit to do cuts, scaling, and many avfilters in >8-bit YUV without
conversion to RGB or 8-bit YUV. With HLG HDR input and color_trc=arib-
std-b67 on the consumer, one can do cuts-only editing, some filtering,
and output HLG. Use AV1 or HEVC (vprofile=main10) with:

mlt_image_format=yuv420p10
pix_fmt=yuv420p10le
color_trc=arib-std-b67

src/modules/avformat/filter_avfilter.c

@@ -809,9 +809,11 @@ static int filter_get_image(mlt_frame frame,
         pdata->avinframe->color_primaries = mlt_properties_get_int(frame_properties,
                                                                    "color_primaries");
         pdata->avinframe->color_trc = mlt_properties_get_int(frame_properties, "color_trc");
-        pdata->avinframe->color_range = mlt_properties_get_int(frame_properties, "full_range")
-                                            ? AVCOL_RANGE_JPEG
-                                            : AVCOL_RANGE_MPEG;
+        // Setting full_range here causes full range input to always be down-converted to limited
+        // range when operating in YUV, regardless of the consumer.color_range -- for each avfilter!
+        // pdata->avinframe->color_range = mlt_properties_get_int(frame_properties, "full_range")
+        //                                     ? AVCOL_RANGE_JPEG
+        //                                     : AVCOL_RANGE_MPEG;
 
         switch (mlt_properties_get_int(frame_properties, "colorspace")) {
         case 240:

src/modules/avformat/filter_swscale.c

@@ -17,6 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
+#include "common.h"
 #include <framework/mlt_factory.h>
 #include <framework/mlt_filter.h>
 #include <framework/mlt_frame.h>
@@ -142,6 +143,28 @@ static int filter_scale(mlt_frame frame,
             goto exit;
         }
 
+        mlt_profile profile = mlt_service_profile(
+            MLT_PRODUCER_SERVICE(mlt_frame_get_original_producer(frame)));
+        int dst_colorspace = profile ? profile->colorspace : 601;
+        int src_colorspace = mlt_properties_get_int(properties, "colorspace");
+        int src_full_range = mlt_properties_get_int(properties, "full_range");
+        const char *dst_color_range = mlt_properties_get(properties, "consumer.color_range");
+        int dst_full_range = mlt_image_full_range(dst_color_range);
+
+        result = mlt_set_luma_transfer(context,
+                                       src_colorspace,
+                                       dst_colorspace,
+                                       src_full_range,
+                                       dst_full_range);
+        if (result < 0) {
+            mlt_log_error(NULL,
+                          "[filter swscale] Setting swscale color options failed with %d (%s)\n",
+                          result,
+                          av_err2str(result));
+            result = 1;
+            goto exit;
+        }
+
         // Setup the input image
         avinframe->width = iwidth;
         avinframe->height = iheight;
@@ -235,6 +258,7 @@ static int filter_scale(mlt_frame frame,
 
         // Now update the MLT frame
         mlt_frame_set_image(frame, outbuf, out_size, mlt_pool_release);
+        mlt_properties_set_int(properties, "full_range", dst_full_range);
 
         // Return the output image
         *image = outbuf;

src/modules/avformat/producer_avformat.c

@@ -603,7 +603,7 @@ static mlt_properties find_default_streams(producer_avformat self)
             else
                 mlt_layout = av_channel_layout_to_mlt(codec_params->channel_layout);
 #endif
-            char *layout = mlt_audio_channel_layout_name(mlt_layout);
+            const char *layout = mlt_audio_channel_layout_name(mlt_layout);
             snprintf(key, sizeof(key), "meta.media.%u.codec.layout", i);
             mlt_properties_set(meta_media, key, layout);
             break;
@@ -783,7 +783,6 @@ static enum AVPixelFormat pick_pix_fmt(enum AVPixelFormat pix_fmt)
 }
 
 static mlt_image_format pick_image_format(enum AVPixelFormat pix_fmt,
-                                          int full_range,
                                           mlt_image_format current_format)
 {
     if (current_format == mlt_image_none || current_format == mlt_image_movit
@@ -819,7 +818,7 @@ static mlt_image_format pick_image_format(enum AVPixelFormat pix_fmt,
             current_format = mlt_image_yuv422;
         }
     }
-    if (pix_fmt == AV_PIX_FMT_BAYER_RGGB16LE || (pix_fmt == AV_PIX_FMT_YUV420P10LE && full_range)) {
+    if (pix_fmt == AV_PIX_FMT_BAYER_RGGB16LE) {
         return mlt_image_rgb;
     } else if (pix_fmt == AV_PIX_FMT_YUVA444P10LE || pix_fmt == AV_PIX_FMT_GBRAP10LE
                || pix_fmt == AV_PIX_FMT_GBRAP12LE) {
@@ -915,9 +914,7 @@ static int get_basic_info(producer_avformat self, mlt_profile profile, const cha
                                                         NULL);
             if (context) {
                 sws_freeContext(context);
-                mlt_image_format format = pick_image_format(pix_fmt,
-                                                            self->full_range,
-                                                            mlt_image_yuv422);
+                mlt_image_format format = pick_image_format(pix_fmt, mlt_image_yuv422);
                 mlt_properties_set_int(properties, "format", format);
             } else
                 error = 1;
@@ -2192,8 +2189,12 @@ static int producer_get_image(mlt_frame frame,
     int got_picture = 0;
     int image_size = 0;
     const char *dst_color_range = mlt_properties_get(frame_properties, "consumer.color_range");
-    int dst_full_range = dst_color_range
-                         && (!strcmp("pc", dst_color_range) || !strcmp("jpeg", dst_color_range));
+    int dst_full_range = mlt_image_full_range(dst_color_range);
+
+    // if depth > 8 libswscale only changes range when scaling, not simple pix_fmt conversion
+    const struct AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(self->video_codec->pix_fmt);
+    if (pix_desc && pix_desc->nb_components > 0 && pix_desc->comp[0].depth > 8)
+        dst_full_range = self->full_range;
 
     mlt_service_lock(MLT_PRODUCER_SERVICE(producer));
     pthread_mutex_lock(&self->video_mutex);
@@ -2291,10 +2292,9 @@ static int producer_get_image(mlt_frame frame,
 #ifdef AVFILTER
     *format = pick_image_format(self->vfilter_out ? av_buffersink_get_format(self->vfilter_out)
                                                   : codec_params->format,
-                                self->full_range,
                                 *format);
 #else
-    *format = pick_image_format(codec_params->format, self->full_range, *format);
+    *format = pick_image_format(codec_params->format, *format);
 #endif
 
     // Duplicate the last image if necessary