From 12d27a0ca506f6ed7f7d4b573fb91cec09b1b912 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 29 Apr 2024 06:53:04 +0200 Subject: [PATCH 01/21] Add support in ogl accelarator for higher color depths --- src/accelerator/accelerator.cpp | 10 +- src/accelerator/accelerator.h | 4 +- src/accelerator/ogl/image/image_kernel.cpp | 23 +++++ src/accelerator/ogl/image/image_mixer.cpp | 65 +++++++++---- src/accelerator/ogl/image/image_mixer.h | 15 ++- src/accelerator/ogl/image/shader.frag | 37 +++---- src/accelerator/ogl/util/device.cpp | 108 +++++++++++---------- src/accelerator/ogl/util/device.h | 7 +- src/accelerator/ogl/util/texture.cpp | 52 ++++++---- src/accelerator/ogl/util/texture.h | 14 +-- src/common/bit_depth.h | 16 +++ src/core/consumer/output.cpp | 17 +++- src/core/frame/frame_factory.h | 4 + src/core/frame/pixel_format.h | 20 ++-- src/core/mixer/image/image_mixer.h | 5 + src/core/mixer/mixer.cpp | 28 ++++-- src/core/mixer/mixer.h | 3 + src/core/video_format.cpp | 3 +- src/shell/server.cpp | 5 +- 19 files changed, 292 insertions(+), 144 deletions(-) create mode 100644 src/common/bit_depth.h diff --git a/src/accelerator/accelerator.cpp b/src/accelerator/accelerator.cpp index 5668553ac6..98196c6d3b 100644 --- a/src/accelerator/accelerator.cpp +++ b/src/accelerator/accelerator.cpp @@ -5,6 +5,8 @@ #include +#include + #include #include @@ -23,10 +25,10 @@ struct accelerator::impl { } - std::unique_ptr create_image_mixer(const int channel_id) + std::unique_ptr create_image_mixer(int channel_id, common::bit_depth depth) { return std::make_unique( - spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size()); + spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size(), depth); } std::shared_ptr get_device() @@ -46,9 +48,9 @@ accelerator::accelerator(const core::video_format_repository format_repository) accelerator::~accelerator() {} -std::unique_ptr accelerator::create_image_mixer(const int channel_id) +std::unique_ptr accelerator::create_image_mixer(const int channel_id, common::bit_depth depth) { - return impl_->create_image_mixer(channel_id); + return impl_->create_image_mixer(channel_id, depth); } std::shared_ptr accelerator::get_device() const diff --git a/src/accelerator/accelerator.h b/src/accelerator/accelerator.h index 5bd67a5f55..f7419d7f99 100644 --- a/src/accelerator/accelerator.h +++ b/src/accelerator/accelerator.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -27,7 +29,7 @@ class accelerator accelerator& operator=(accelerator&) = delete; - std::unique_ptr create_image_mixer(int channel_id); + std::unique_ptr create_image_mixer(int channel_id, common::bit_depth depth); std::shared_ptr get_device() const; diff --git a/src/accelerator/ogl/image/image_kernel.cpp b/src/accelerator/ogl/image/image_kernel.cpp index 155d574e8d..40d0ede789 100644 --- a/src/accelerator/ogl/image/image_kernel.cpp +++ b/src/accelerator/ogl/image/image_kernel.cpp @@ -81,6 +81,22 @@ double hypotenuse(double x1, double y1, double x2, double y2) return std::sqrt(x * x + y * y); } +double get_precision_factor(common::bit_depth depth) +{ + switch (depth) { + case common::bit_depth::bit8: + return 1.0; + case common::bit_depth::bit10: + return 64.0; + case common::bit_depth::bit12: + return 16.0; + case common::bit_depth::bit16: + return 1.0; + default: + return 1.0; + } +} + double calc_q(double close_diagonal, double distant_diagonal) { return (close_diagonal + distant_diagonal) / distant_diagonal; @@ -221,10 +237,13 @@ struct image_kernel::impl return; } + double precision_factor[4] = {1, 1, 1, 1}; + // Bind textures for (int n = 0; n < params.textures.size(); ++n) { params.textures[n]->bind(n); + precision_factor[n] = get_precision_factor(params.textures[n]->depth()); } if (params.local_key) { @@ -243,6 +262,10 @@ struct image_kernel::impl shader_->set("plane[1]", texture_id::plane1); shader_->set("plane[2]", texture_id::plane2); shader_->set("plane[3]", texture_id::plane3); + shader_->set("precision_factor[0]", precision_factor[0]); + shader_->set("precision_factor[1]", precision_factor[1]); + shader_->set("precision_factor[2]", precision_factor[2]); + shader_->set("precision_factor[3]", precision_factor[3]); shader_->set("local_key", texture_id::local_key); shader_->set("layer_key", texture_id::layer_key); shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0); diff --git a/src/accelerator/ogl/image/image_mixer.cpp b/src/accelerator/ogl/image/image_mixer.cpp index e5add850a0..e34f299503 100644 --- a/src/accelerator/ogl/image/image_mixer.cpp +++ b/src/accelerator/ogl/image/image_mixer.cpp @@ -27,6 +27,7 @@ #include "../util/texture.h" #include +#include #include #include @@ -70,12 +71,14 @@ class image_renderer spl::shared_ptr ogl_; image_kernel kernel_; const size_t max_frame_size_; + common::bit_depth depth_; public: - explicit image_renderer(const spl::shared_ptr& ogl, const size_t max_frame_size) + explicit image_renderer(const spl::shared_ptr& ogl, const size_t max_frame_size, common::bit_depth depth) : ogl_(ogl) , kernel_(ogl_) , max_frame_size_(max_frame_size) + , depth_(depth) { } @@ -88,7 +91,7 @@ class image_renderer } return flatten(ogl_->dispatch_async([=]() mutable -> std::shared_future> { - auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4); + auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_); draw(target_texture, std::move(layers), format_desc); @@ -96,6 +99,8 @@ class image_renderer })); } + common::bit_depth depth() const { return depth_; } + private: void draw(std::shared_ptr& target_texture, std::vector layers, @@ -121,7 +126,7 @@ class image_renderer std::shared_ptr local_mix_texture; if (layer.blend_mode != core::blend_mode::normal) { - auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4); + auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_); for (auto& item : layer.items) draw(layer_texture, @@ -168,9 +173,9 @@ class image_renderer } if (item.transform.is_key) { - local_key_texture = local_key_texture - ? local_key_texture - : ogl_->create_texture(target_texture->width(), target_texture->height(), 1); + local_key_texture = + local_key_texture ? local_key_texture + : ogl_->create_texture(target_texture->width(), target_texture->height(), 1, depth_); draw_params.background = local_key_texture; draw_params.local_key = nullptr; @@ -178,9 +183,9 @@ class image_renderer kernel_.draw(std::move(draw_params)); } else if (item.transform.is_mix) { - local_mix_texture = local_mix_texture - ? local_mix_texture - : ogl_->create_texture(target_texture->width(), target_texture->height(), 4); + local_mix_texture = + local_mix_texture ? local_mix_texture + : ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_); draw_params.background = local_mix_texture; draw_params.local_key = std::move(local_key_texture); @@ -210,7 +215,7 @@ class image_renderer draw_params draw_params; draw_params.pix_desc.format = core::pixel_format::bgra; draw_params.pix_desc.planes = { - core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4)}; + core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4, source_buffer->depth())}; draw_params.textures = {spl::make_shared_ptr(source_buffer)}; draw_params.transform = core::image_transform(); draw_params.blend_mode = blend_mode; @@ -232,9 +237,9 @@ struct image_mixer::impl std::vector layer_stack_; public: - impl(const spl::shared_ptr& ogl, const int channel_id, const size_t max_frame_size) + impl(const spl::shared_ptr& ogl, const int channel_id, const size_t max_frame_size, common::bit_depth depth) : ogl_(ogl) - , renderer_(ogl, max_frame_size) + , renderer_(ogl, max_frame_size, depth) , transform_stack_(1) { CASPAR_LOG(info) << L"Initialized OpenGL Accelerated GPU Image Mixer for channel " << channel_id; @@ -281,7 +286,8 @@ struct image_mixer::impl item.textures.emplace_back(ogl_->copy_async(frame.image_data(n), item.pix_desc.planes[n].width, item.pix_desc.planes[n].height, - item.pix_desc.planes[n].stride)); + item.pix_desc.planes[n].stride, + item.pix_desc.planes[n].depth)); } } @@ -300,10 +306,18 @@ struct image_mixer::impl } core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override + { + return create_frame(tag, desc, common::bit_depth::bit8); + } + + core::mutable_frame + create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth) override { std::vector> image_data; for (auto& plane : desc.planes) { - image_data.push_back(ogl_->create_array(plane.size)); + image_data.push_back(ogl_->create_array(plane.size, + plane.depth == common::bit_depth::bit8 ? common::bit_depth::bit8 + : common::bit_depth::bit16)); } std::weak_ptr weak_self = shared_from_this(); @@ -319,16 +333,24 @@ struct image_mixer::impl } std::vector textures; for (int n = 0; n < static_cast(desc.planes.size()); ++n) { - textures.emplace_back(self->ogl_->copy_async( - image_data[n], desc.planes[n].width, desc.planes[n].height, desc.planes[n].stride)); + textures.emplace_back(self->ogl_->copy_async(image_data[n], + desc.planes[n].width, + desc.planes[n].height, + desc.planes[n].stride, + desc.planes[n].depth)); } return std::make_shared(std::move(textures)); }); } + + common::bit_depth depth() const { return renderer_.depth(); } }; -image_mixer::image_mixer(const spl::shared_ptr& ogl, const int channel_id, const size_t max_frame_size) - : impl_(std::make_unique(ogl, channel_id, max_frame_size)) +image_mixer::image_mixer(const spl::shared_ptr& ogl, + const int channel_id, + const size_t max_frame_size, + common::bit_depth depth) + : impl_(std::make_unique(ogl, channel_id, max_frame_size, depth)) { } image_mixer::~image_mixer() {} @@ -343,5 +365,12 @@ core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel { return impl_->create_frame(tag, desc); } +core::mutable_frame +image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth) +{ + return impl_->create_frame(tag, desc, depth); +} + +common::bit_depth image_mixer::depth() const { return impl_->depth(); } }}} // namespace caspar::accelerator::ogl diff --git a/src/accelerator/ogl/image/image_mixer.h b/src/accelerator/ogl/image/image_mixer.h index c9034238d8..609a7e8d11 100644 --- a/src/accelerator/ogl/image/image_mixer.h +++ b/src/accelerator/ogl/image/image_mixer.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include @@ -36,7 +37,10 @@ namespace caspar { namespace accelerator { namespace ogl { class image_mixer final : public core::image_mixer { public: - image_mixer(const spl::shared_ptr& ogl, int channel_id, const size_t max_frame_size); + image_mixer(const spl::shared_ptr& ogl, + int channel_id, + const size_t max_frame_size, + common::bit_depth depth); image_mixer(const image_mixer&) = delete; ~image_mixer(); @@ -45,12 +49,15 @@ class image_mixer final : public core::image_mixer std::future> operator()(const core::video_format_desc& format_desc) override; core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override; + core::mutable_frame + create_frame(const void* video_stream_tag, const core::pixel_format_desc& desc, common::bit_depth depth) override; // core::image_mixer - void push(const core::frame_transform& frame) override; - void visit(const core::const_frame& frame) override; - void pop() override; + void push(const core::frame_transform& frame) override; + void visit(const core::const_frame& frame) override; + void pop() override; + common::bit_depth depth() const override; private: struct impl; diff --git a/src/accelerator/ogl/image/shader.frag b/src/accelerator/ogl/image/shader.frag index 93d31d13f2..e277d792f7 100644 --- a/src/accelerator/ogl/image/shader.frag +++ b/src/accelerator/ogl/image/shader.frag @@ -23,6 +23,7 @@ uniform float max_input; uniform float gamma; uniform float min_output; uniform float max_output; +uniform float precision_factor[4]; uniform bool csb; uniform float brt; @@ -476,44 +477,44 @@ vec4 get_rgba_color() switch(pixel_format) { case 0: //gray - return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).rrr, 1.0); + return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).rrr * precision_factor[0], 1.0); case 1: //bgra, - return get_sample(plane[0], TexCoord.st / TexCoord.q).bgra; + return get_sample(plane[0], TexCoord.st / TexCoord.q).bgra * precision_factor[0]; case 2: //rgba, - return get_sample(plane[0], TexCoord.st / TexCoord.q).rgba; + return get_sample(plane[0], TexCoord.st / TexCoord.q).rgba * precision_factor[0]; case 3: //argb, - return get_sample(plane[0], TexCoord.st / TexCoord.q).argb; + return get_sample(plane[0], TexCoord.st / TexCoord.q).argb * precision_factor[0]; case 4: //abgr, - return get_sample(plane[0], TexCoord.st / TexCoord.q).gbar; + return get_sample(plane[0], TexCoord.st / TexCoord.q).gbar * precision_factor[0]; case 5: //ycbcr, { - float y = get_sample(plane[0], TexCoord.st / TexCoord.q).r; - float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).r; - float cr = get_sample(plane[2], TexCoord.st / TexCoord.q).r; + float y = get_sample(plane[0], TexCoord.st / TexCoord.q).r * precision_factor[0]; + float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).r * precision_factor[1]; + float cr = get_sample(plane[2], TexCoord.st / TexCoord.q).r * precision_factor[2]; return ycbcra_to_rgba(y, cb, cr, 1.0); } case 6: //ycbcra { - float y = get_sample(plane[0], TexCoord.st / TexCoord.q).r; - float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).r; - float cr = get_sample(plane[2], TexCoord.st / TexCoord.q).r; - float a = get_sample(plane[3], TexCoord.st / TexCoord.q).r; + float y = get_sample(plane[0], TexCoord.st / TexCoord.q).r * precision_factor[0]; + float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).r * precision_factor[1]; + float cr = get_sample(plane[2], TexCoord.st / TexCoord.q).r * precision_factor[2]; + float a = get_sample(plane[3], TexCoord.st / TexCoord.q).r * precision_factor[3]; return ycbcra_to_rgba(y, cb, cr, a); } case 7: //luma { - vec3 y3 = get_sample(plane[0], TexCoord.st / TexCoord.q).rrr; + vec3 y3 = get_sample(plane[0], TexCoord.st / TexCoord.q).rrr * precision_factor[0]; return vec4((y3-0.065)/0.859, 1.0); } case 8: //bgr, - return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).bgr, 1.0); + return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).bgr * precision_factor[0], 1.0); case 9: //rgb, - return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).rgb, 1.0); + return vec4(get_sample(plane[0], TexCoord.st / TexCoord.q).rgb * precision_factor[0], 1.0); case 10: // uyvy { - float y = get_sample(plane[0], TexCoord.st / TexCoord.q).g; - float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).b; - float cr = get_sample(plane[1], TexCoord.st / TexCoord.q).r; + float y = get_sample(plane[0], TexCoord.st / TexCoord.q).g * precision_factor[0]; + float cb = get_sample(plane[1], TexCoord.st / TexCoord.q).b * precision_factor[1]; + float cr = get_sample(plane[1], TexCoord.st / TexCoord.q).r * precision_factor[1]; return ycbcra_to_rgba(y, cb, cr, 1.0); } } diff --git a/src/accelerator/ogl/util/device.cpp b/src/accelerator/ogl/util/device.cpp index 0576153f7f..ec3220e0fd 100644 --- a/src/accelerator/ogl/util/device.cpp +++ b/src/accelerator/ogl/util/device.cpp @@ -62,8 +62,8 @@ struct device::impl : public std::enable_shared_from_this sf::Context device_; - std::array, 4> device_pools_; - std::array, 2> host_pools_; + std::array, 4>, 2> device_pools_; + std::array, 2> host_pools_; using sync_queue_t = tbb::concurrent_bounded_queue>; @@ -86,7 +86,7 @@ struct device::impl : public std::enable_shared_from_this device_.setActive(true); auto err = glewInit(); - if (err != GLEW_OK && err != GLEW_ERROR_NO_GLX_DISPLAY) { + if (err != GLEW_OK && err != 4) { // GLEW_ERROR_NO_GLX_DISPLAY std::stringstream str; str << "Failed to initialize GLEW (" << (int)err << "): " << glewGetErrorString(err) << std::endl; CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info(str.str())); @@ -133,8 +133,9 @@ struct device::impl : public std::enable_shared_from_this for (auto& pool : host_pools_) pool.clear(); - for (auto& pool : device_pools_) - pool.clear(); + for (auto& pools : device_pools_) + for (auto& pool : pools) + pool.clear(); sync_queue_.clear(); @@ -173,17 +174,19 @@ struct device::impl : public std::enable_shared_from_this std::wstring version() { return version_; } - std::shared_ptr create_texture(int width, int height, int stride, bool clear) + std::shared_ptr create_texture(int width, int height, int stride, common::bit_depth depth, bool clear) { CASPAR_VERIFY(stride > 0 && stride < 5); CASPAR_VERIFY(width > 0 && height > 0); + auto depth_pool_index = depth == common::bit_depth::bit8 ? 0 : 1; + // TODO (perf) Shared pool. - auto pool = &device_pools_[stride - 1][(width << 16 & 0xFFFF0000) | (height & 0x0000FFFF)]; + auto pool = &device_pools_[depth_pool_index][stride - 1][(width << 16 & 0xFFFF0000) | (height & 0x0000FFFF)]; std::shared_ptr tex; if (!pool->try_pop(tex)) { - tex = std::make_shared(width, height, stride); + tex = std::make_shared(width, height, stride, depth); } if (clear) { @@ -214,15 +217,16 @@ struct device::impl : public std::enable_shared_from_this }); } - array create_array(int size) + array create_array(int count, common::bit_depth depth) { - auto buf = create_buffer(size, true); - auto ptr = reinterpret_cast(buf->data()); + auto bytes_per_pixel = depth == common::bit_depth::bit8 ? 1 : 2; + auto buf = create_buffer(count * bytes_per_pixel, true); + auto ptr = reinterpret_cast(buf->data()); return array(ptr, buf->size(), buf); } std::future> - copy_async(const array& source, int width, int height, int stride) + copy_async(const array& source, int width, int height, int stride, common::bit_depth depth) { return dispatch_async([=] { std::shared_ptr buf; @@ -236,7 +240,7 @@ struct device::impl : public std::enable_shared_from_this std::memcpy(buf->data(), source.data(), source.size()); } - auto tex = create_texture(width, height, stride, false); + auto tex = create_texture(width, height, stride, depth, false); tex->copy_from(*buf); // TODO (perf) save tex on source return tex; @@ -284,10 +288,11 @@ struct device::impl : public std::enable_shared_from_this } #ifdef WIN32 - std::future> copy_async(GLuint source, int width, int height, int stride) + /* Unused? */ + std::future> copy_async(GLuint source, int width, int height, int stride, common::bit_depth depth) { return spawn_async([=](yield_context yield) { - auto tex = create_texture(width, height, stride, false); + auto tex = create_texture(width, height, stride, depth, false); tex->copy_from(source); @@ -323,32 +328,35 @@ struct device::impl : public std::enable_shared_from_this size_t total_pooled_device_buffer_count = 0; for (size_t i = 0; i < device_pools_.size(); ++i) { - auto& pools = device_pools_.at(i); - bool mipmapping = i > 3; - auto stride = mipmapping ? i - 3 : i + 1; - - for (auto& pool : pools) { - auto width = pool.first >> 16; - auto height = pool.first & 0x0000FFFF; - auto size = width * height * stride; - auto count = pool.second.size(); - - if (count == 0) - continue; - - boost::property_tree::wptree pool_info; - - pool_info.add(L"stride", stride); - pool_info.add(L"mipmapping", mipmapping); - pool_info.add(L"width", width); - pool_info.add(L"height", height); - pool_info.add(L"size", size); - pool_info.add(L"count", count); - - total_pooled_device_buffer_size += size * count; - total_pooled_device_buffer_count += count; - - pooled_device_buffers.add_child(L"device_buffer_pool", pool_info); + auto& depth_pools = device_pools_.at(i); + for (size_t j = 0; j < depth_pools.size(); ++j) { + auto& pools = depth_pools.at(j); + bool mipmapping = j > 3; + auto stride = mipmapping ? j - 3 : j + 1; + + for (auto& pool : pools) { + auto width = pool.first >> 16; + auto height = pool.first & 0x0000FFFF; + auto size = width * height * stride; + auto count = pool.second.size(); + + if (count == 0) + continue; + + boost::property_tree::wptree pool_info; + + pool_info.add(L"stride", stride); + pool_info.add(L"mipmapping", mipmapping); + pool_info.add(L"width", width); + pool_info.add(L"height", height); + pool_info.add(L"size", size); + pool_info.add(L"count", count); + + total_pooled_device_buffer_size += size * count; + total_pooled_device_buffer_count += count; + + pooled_device_buffers.add_child(L"device_buffer_pool", pool_info); + } } } @@ -403,9 +411,11 @@ struct device::impl : public std::enable_shared_from_this CASPAR_LOG(info) << " ogl: Running GC."; try { - for (auto& pools : device_pools_) { - for (auto& pool : pools) - pool.second.clear(); + for (auto& depth_pools : device_pools_) { + for (auto& pools : depth_pools) { + for (auto& pool : pools) + pool.second.clear(); + } } for (auto& pools : host_pools_) { for (auto& pool : pools) @@ -423,15 +433,15 @@ device::device() { } device::~device() {} -std::shared_ptr device::create_texture(int width, int height, int stride) +std::shared_ptr device::create_texture(int width, int height, int stride, common::bit_depth depth) { - return impl_->create_texture(width, height, stride, true); + return impl_->create_texture(width, height, stride, depth, true); } -array device::create_array(int size) { return impl_->create_array(size); } +array device::create_array(int size, common::bit_depth depth) { return impl_->create_array(size, depth); } std::future> -device::copy_async(const array& source, int width, int height, int stride) +device::copy_async(const array& source, int width, int height, int stride, common::bit_depth depth) { - return impl_->copy_async(source, width, height, stride); + return impl_->copy_async(source, width, height, stride, depth); } std::future> device::copy_async(const std::shared_ptr& source) { diff --git a/src/accelerator/ogl/util/device.h b/src/accelerator/ogl/util/device.h index d7f1cef1a4..e7c6982715 100644 --- a/src/accelerator/ogl/util/device.h +++ b/src/accelerator/ogl/util/device.h @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -45,11 +46,11 @@ class device final device& operator=(const device&) = delete; - std::shared_ptr create_texture(int width, int height, int stride); - array create_array(int size); + std::shared_ptr create_texture(int width, int height, int stride, common::bit_depth depth); + array create_array(int size, common::bit_depth depth); std::future> - copy_async(const array& source, int width, int height, int stride); + copy_async(const array& source, int width, int height, int stride, common::bit_depth depth); std::future> copy_async(const std::shared_ptr& source); template auto dispatch_async(Func&& func) diff --git a/src/accelerator/ogl/util/texture.cpp b/src/accelerator/ogl/util/texture.cpp index 8682d060ef..69b8c2e225 100644 --- a/src/accelerator/ogl/util/texture.cpp +++ b/src/accelerator/ogl/util/texture.cpp @@ -22,40 +22,45 @@ #include "buffer.h" +#include #include #include namespace caspar { namespace accelerator { namespace ogl { -static GLenum FORMAT[] = {0, GL_RED, GL_RG, GL_BGR, GL_BGRA}; -static GLenum INTERNAL_FORMAT[] = {0, GL_R8, GL_RG8, GL_RGB8, GL_RGBA8}; -static GLenum TYPE[] = {0, GL_UNSIGNED_BYTE, GL_UNSIGNED_BYTE, GL_UNSIGNED_BYTE, GL_UNSIGNED_INT_8_8_8_8_REV}; +static GLenum FORMAT[] = {0, GL_RED, GL_RG, GL_BGR, GL_BGRA}; +static GLenum INTERNAL_FORMAT[][5] = {{0, GL_R8, GL_RG8, GL_RGB8, GL_RGBA8}, {0, GL_R16, GL_RG16, GL_RGB16, GL_RGBA16}}; +static GLenum TYPE[][5] = {{0, GL_UNSIGNED_BYTE, GL_UNSIGNED_BYTE, GL_UNSIGNED_BYTE, GL_UNSIGNED_INT_8_8_8_8_REV}, + {0, GL_UNSIGNED_SHORT, GL_UNSIGNED_SHORT, GL_UNSIGNED_SHORT, GL_UNSIGNED_SHORT}}; struct texture::impl { - GLuint id_ = 0; - GLsizei width_ = 0; - GLsizei height_ = 0; - GLsizei stride_ = 0; - GLsizei size_ = 0; + GLuint id_ = 0; + GLsizei width_ = 0; + GLsizei height_ = 0; + GLsizei stride_ = 0; + GLsizei size_ = 0; + common::bit_depth depth_; impl(const impl&) = delete; impl& operator=(const impl&) = delete; public: - impl(int width, int height, int stride) + impl(int width, int height, int stride, common::bit_depth depth) : width_(width) , height_(height) , stride_(stride) - , size_(width * height * stride) + , depth_(depth) + , size_(width * height * stride * (depth == common::bit_depth::bit8 ? 1 : 2)) { GL(glCreateTextures(GL_TEXTURE_2D, 1, &id_)); GL(glTextureParameteri(id_, GL_TEXTURE_MIN_FILTER, GL_LINEAR)); GL(glTextureParameteri(id_, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); GL(glTextureParameteri(id_, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); GL(glTextureParameteri(id_, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); - GL(glTextureStorage2D(id_, 1, INTERNAL_FORMAT[stride_], width_, height_)); + GL(glTextureStorage2D( + id_, 1, INTERNAL_FORMAT[depth_ == common::bit_depth::bit8 ? 0 : 1][stride_], width_, height_)); } ~impl() { glDeleteTextures(1, &id_); } @@ -72,7 +77,10 @@ struct texture::impl void attach() { GL(glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0 + 0, GL_TEXTURE_2D, id_, 0)); } - void clear() { GL(glClearTexImage(id_, 0, FORMAT[stride_], TYPE[stride_], nullptr)); } + void clear() + { + GL(glClearTexImage(id_, 0, FORMAT[stride_], TYPE[depth_ == common::bit_depth::bit8 ? 0 : 1][stride_], nullptr)); + } #ifdef WIN32 void copy_from(int texture_id) @@ -92,7 +100,15 @@ struct texture::impl glPixelStorei(GL_UNPACK_ALIGNMENT, 4); } - GL(glTextureSubImage2D(id_, 0, 0, 0, width_, height_, FORMAT[stride_], TYPE[stride_], nullptr)); + GL(glTextureSubImage2D(id_, + 0, + 0, + 0, + width_, + height_, + FORMAT[stride_], + TYPE[depth_ == common::bit_depth::bit8 ? 0 : 1][stride_], + nullptr)); src.unbind(); } @@ -100,13 +116,14 @@ struct texture::impl void copy_to(buffer& dst) { dst.bind(); - GL(glGetTextureImage(id_, 0, FORMAT[stride_], TYPE[stride_], size_, nullptr)); + GL(glGetTextureImage( + id_, 0, FORMAT[stride_], TYPE[depth_ == common::bit_depth::bit8 ? 0 : 1][stride_], size_, nullptr)); dst.unbind(); } }; -texture::texture(int width, int height, int stride) - : impl_(new impl(width, height, stride)) +texture::texture(int width, int height, int stride, common::bit_depth depth) + : impl_(new impl(width, height, stride, depth)) { } texture::texture(texture&& other) @@ -131,7 +148,8 @@ void texture::copy_to(buffer& dest) { impl_->copy_to(dest); } int texture::width() const { return impl_->width_; } int texture::height() const { return impl_->height_; } int texture::stride() const { return impl_->stride_; } -int texture::size() const { return impl_->width_ * impl_->height_ * impl_->stride_; } +common::bit_depth texture::depth() const { return impl_->depth_; } +int texture::size() const { return impl_->size_; } int texture::id() const { return impl_->id_; } }}} // namespace caspar::accelerator::ogl diff --git a/src/accelerator/ogl/util/texture.h b/src/accelerator/ogl/util/texture.h index ccdca84250..ff2c117f73 100644 --- a/src/accelerator/ogl/util/texture.h +++ b/src/accelerator/ogl/util/texture.h @@ -21,6 +21,7 @@ #pragma once +#include #include namespace caspar { namespace accelerator { namespace ogl { @@ -28,7 +29,7 @@ namespace caspar { namespace accelerator { namespace ogl { class texture final { public: - texture(int width, int height, int stride); + texture(int width, int height, int stride, common::bit_depth depth = common::bit_depth::bit8); texture(const texture&) = delete; texture(texture&& other); ~texture(); @@ -47,11 +48,12 @@ class texture final void bind(int index); void unbind(); - int width() const; - int height() const; - int stride() const; - int size() const; - int id() const; + int width() const; + int height() const; + int stride() const; + common::bit_depth depth() const; + int size() const; + int id() const; private: struct impl; diff --git a/src/common/bit_depth.h b/src/common/bit_depth.h new file mode 100644 index 0000000000..389f9a6431 --- /dev/null +++ b/src/common/bit_depth.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace caspar { namespace common { + +enum class bit_depth : uint8_t +{ + bit8 = 0, + bit10, + bit12, + bit14, + bit16, +}; + +}} // namespace caspar::common \ No newline at end of file diff --git a/src/core/consumer/output.cpp b/src/core/consumer/output.cpp index e33628fcb6..e97c5909c8 100644 --- a/src/core/consumer/output.cpp +++ b/src/core/consumer/output.cpp @@ -23,7 +23,9 @@ #include "frame_consumer.h" #include "../frame/frame.h" +#include "../frame/pixel_format.h" +#include #include #include #include @@ -105,14 +107,21 @@ struct output::impl return; } - if (input_frame1.size() != format_desc_.size) { + const auto bytesPerComponent1 = + input_frame1.pixel_format_desc().planes.at(0).depth == common::bit_depth::bit8 ? 1 : 2; + if (input_frame1.size() != format_desc_.size * bytesPerComponent1) { CASPAR_LOG(warning) << print() << L" Invalid input frame size."; return; } - if (input_frame2 && input_frame2.size() != format_desc_.size) { - CASPAR_LOG(warning) << print() << L" Invalid input frame size."; - return; + if (input_frame2) { + const auto bytesPerComponent2 = + input_frame2.pixel_format_desc().planes.at(0).depth == common::bit_depth::bit8 ? 1 : 2; + + if(input_frame2.size() != format_desc_.size * bytesPerComponent2) { + CASPAR_LOG(warning) << print() << L" Invalid input frame size."; + return; + } } decltype(consumers_) consumers; diff --git a/src/core/frame/frame_factory.h b/src/core/frame/frame_factory.h index 965b7e9b40..c0320eea4f 100644 --- a/src/core/frame/frame_factory.h +++ b/src/core/frame/frame_factory.h @@ -21,6 +21,8 @@ #pragma once +#include + namespace caspar { namespace core { class frame_factory @@ -33,6 +35,8 @@ class frame_factory frame_factory(const frame_factory&) = delete; virtual class mutable_frame create_frame(const void* video_stream_tag, const struct pixel_format_desc& desc) = 0; + virtual class mutable_frame + create_frame(const void* video_stream_tag, const struct pixel_format_desc& desc, common::bit_depth depth) = 0; }; }} // namespace caspar::core diff --git a/src/core/frame/pixel_format.h b/src/core/frame/pixel_format.h index 007e1b5082..338e9d85fc 100644 --- a/src/core/frame/pixel_format.h +++ b/src/core/frame/pixel_format.h @@ -23,6 +23,8 @@ #include +#include + namespace caspar { namespace core { enum class pixel_format @@ -46,20 +48,22 @@ struct pixel_format_desc final { struct plane { - int linesize = 0; - int width = 0; - int height = 0; - int size = 0; - int stride = 0; + int linesize = 0; + int width = 0; + int height = 0; + int size = 0; + int stride = 0; + common::bit_depth depth = common::bit_depth::bit8; plane() = default; - plane(int width, int height, int stride) - : linesize(width * stride) + plane(int width, int height, int stride, common::bit_depth depth = common::bit_depth::bit8) + : linesize(width * stride * (depth == common::bit_depth::bit8 ? 1 : 2)) , width(width) , height(height) - , size(width * height * stride) + , size(width * height * stride * (depth == common::bit_depth::bit8 ? 1 : 2)) , stride(stride) + , depth(depth) { } }; diff --git a/src/core/mixer/image/image_mixer.h b/src/core/mixer/image/image_mixer.h index 0922e38261..795bec1bcb 100644 --- a/src/core/mixer/image/image_mixer.h +++ b/src/core/mixer/image/image_mixer.h @@ -48,6 +48,11 @@ class image_mixer virtual std::future> operator()(const struct video_format_desc& format_desc) = 0; class mutable_frame create_frame(const void* tag, const struct pixel_format_desc& desc) override = 0; + class mutable_frame create_frame(const void* video_stream_tag, + const struct pixel_format_desc& desc, + common::bit_depth depth) override = 0; + + virtual common::bit_depth depth() const = 0; }; }} // namespace caspar::core diff --git a/src/core/mixer/mixer.cpp b/src/core/mixer/mixer.cpp index 1095c2108a..fbf606b90d 100644 --- a/src/core/mixer/mixer.cpp +++ b/src/core/mixer/mixer.cpp @@ -35,6 +35,7 @@ #include #include +#include #include #include @@ -72,15 +73,22 @@ struct mixer::impl state_["audio"] = audio_mixer_.state(); - buffer_.push(std::async( - std::launch::deferred, - [image = std::move(image), audio = std::move(audio), graph = graph_, format_desc, tag = this]() mutable { - auto desc = pixel_format_desc(pixel_format::bgra); - desc.planes.push_back(pixel_format_desc::plane(format_desc.width, format_desc.height, 4)); - std::vector> image_data; - image_data.emplace_back(std::move(image.get())); - return const_frame(std::move(image_data), std::move(audio), desc); - })); + auto depth = image_mixer_->depth(); + + buffer_.push(std::async(std::launch::deferred, + [image = std::move(image), + audio = std::move(audio), + graph = graph_, + depth, + format_desc, + tag = this]() mutable { + auto desc = pixel_format_desc(pixel_format::bgra); + desc.planes.push_back( + pixel_format_desc::plane(format_desc.width, format_desc.height, 4, depth)); + std::vector> image_data; + image_data.emplace_back(std::move(image.get())); + return const_frame(std::move(image_data), std::move(audio), desc); + })); if (buffer_.size() <= format_desc.field_count) { return const_frame{}; @@ -111,4 +119,6 @@ mutable_frame mixer::create_frame(const void* tag, const pixel_format_desc& desc return impl_->image_mixer_->create_frame(tag, desc); } core::monitor::state mixer::state() const { return impl_->state_; } + +common::bit_depth mixer::depth() const { return impl_->image_mixer_->depth(); } }} // namespace caspar::core diff --git a/src/core/mixer/mixer.h b/src/core/mixer/mixer.h index b50ec45ed8..2473c0ead1 100644 --- a/src/core/mixer/mixer.h +++ b/src/core/mixer/mixer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -50,6 +51,8 @@ class mixer final core::monitor::state state() const; + common::bit_depth depth() const; + private: struct impl; spl::shared_ptr impl_; diff --git a/src/core/video_format.cpp b/src/core/video_format.cpp index baa897540c..871cbd7c2e 100644 --- a/src/core/video_format.cpp +++ b/src/core/video_format.cpp @@ -168,7 +168,8 @@ struct video_format_repository::impl max = f.second.size; } - return max; + const size_t MaxBytesPerColor = 2; + return max * MaxBytesPerColor; } }; diff --git a/src/shell/server.cpp b/src/shell/server.cpp index 09ea2d6f5e..d5749b0961 100644 --- a/src/shell/server.cpp +++ b/src/shell/server.cpp @@ -24,6 +24,7 @@ #include +#include #include #include #include @@ -111,7 +112,7 @@ struct server::impl spl::shared_ptr consumer_registry_; std::function shutdown_server_now_; - impl(const impl&) = delete; + impl(const impl&) = delete; impl& operator=(const impl&) = delete; explicit impl(std::function shutdown_server_now) @@ -263,7 +264,7 @@ struct server::impl auto channel = spl::make_shared(channel_id, format_desc, - accelerator_.create_image_mixer(channel_id), + accelerator_.create_image_mixer(channel_id, common::bit_depth::bit8), [channel_id, weak_client](core::monitor::state channel_state) { monitor::state state; state[""]["channel"][channel_id] = channel_state; From 3e1f1b17cbccd8bc7d0c61147d7ef6d9b03e8e17 Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:37:58 +0000 Subject: [PATCH 02/21] Make consumer factories aware of color depth --- src/core/consumer/frame_consumer.cpp | 10 +++++--- src/core/consumer/frame_consumer.h | 25 +++++++++++-------- .../artnet/consumer/artnet_consumer.cpp | 6 ++++- src/modules/artnet/consumer/artnet_consumer.h | 4 ++- .../bluefish/consumer/bluefish_consumer.cpp | 13 +++++++--- .../bluefish/consumer/bluefish_consumer.h | 7 ++++-- .../decklink/consumer/decklink_consumer.cpp | 12 +++++++-- .../decklink/consumer/decklink_consumer.h | 7 ++++-- .../ffmpeg/consumer/ffmpeg_consumer.cpp | 13 ++++++++-- src/modules/ffmpeg/consumer/ffmpeg_consumer.h | 7 ++++-- src/modules/image/consumer/image_consumer.cpp | 7 +++++- src/modules/image/consumer/image_consumer.h | 4 ++- .../newtek/consumer/newtek_ndi_consumer.cpp | 15 +++++++++-- .../newtek/consumer/newtek_ndi_consumer.h | 7 ++++-- src/modules/oal/consumer/oal_consumer.cpp | 6 +++-- src/modules/oal/consumer/oal_consumer.h | 7 ++++-- .../screen/consumer/screen_consumer.cpp | 13 ++++++++-- src/modules/screen/consumer/screen_consumer.h | 7 ++++-- src/protocol/amcp/AMCPCommandsImpl.cpp | 20 ++++++++++----- src/shell/server.cpp | 8 ++++-- 20 files changed, 146 insertions(+), 52 deletions(-) diff --git a/src/core/consumer/frame_consumer.cpp b/src/core/consumer/frame_consumer.cpp index 1c410edc37..5e3accbdc9 100644 --- a/src/core/consumer/frame_consumer.cpp +++ b/src/core/consumer/frame_consumer.cpp @@ -163,7 +163,8 @@ class print_consumer_proxy : public frame_consumer spl::shared_ptr frame_consumer_registry::create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) const + const std::vector>& channels, + common::bit_depth depth) const { if (params.empty()) CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("params cannot be empty")); @@ -173,7 +174,7 @@ frame_consumer_registry::create_consumer(const std::vector& if (!std::any_of( consumer_factories.begin(), consumer_factories.end(), [&](const consumer_factory_t& factory) -> bool { try { - consumer = factory(params, format_repository, channels); + consumer = factory(params, format_repository, channels, depth); } catch (...) { CASPAR_LOG_CURRENT_EXCEPTION(); } @@ -189,7 +190,8 @@ spl::shared_ptr frame_consumer_registry::create_consumer(const std::wstring& element_name, const boost::property_tree::wptree& element, const core::video_format_repository& format_repository, - const std::vector>& channels) const + const std::vector>& channels, + common::bit_depth depth) const { auto& preconfigured_consumer_factories = impl_->preconfigured_consumer_factories; auto found = preconfigured_consumer_factories.find(element_name); @@ -199,7 +201,7 @@ frame_consumer_registry::create_consumer(const std::wstring& << msg_info(L"No consumer factory registered for element name " + element_name)); return spl::make_shared( - spl::make_shared(found->second(element, format_repository, channels))); + spl::make_shared(found->second(element, format_repository, channels, depth))); } const spl::shared_ptr& frame_consumer::empty() diff --git a/src/core/consumer/frame_consumer.h b/src/core/consumer/frame_consumer.h index 55f8dbd7e1..c3769f8c4d 100644 --- a/src/core/consumer/frame_consumer.h +++ b/src/core/consumer/frame_consumer.h @@ -24,6 +24,7 @@ #include "../fwd.h" #include "../monitor/monitor.h" +#include #include #include @@ -62,11 +63,13 @@ class frame_consumer using consumer_factory_t = std::function(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels)>; + const std::vector>& channels, + common::bit_depth depth)>; using preconfigured_consumer_factory_t = std::function(const boost::property_tree::wptree& element, const core::video_format_repository& format_repository, - const std::vector>& channels)>; + const std::vector>& channels, + common::bit_depth depth)>; class frame_consumer_registry { @@ -75,15 +78,15 @@ class frame_consumer_registry void register_consumer_factory(const std::wstring& name, const consumer_factory_t& factory); void register_preconfigured_consumer_factory(const std::wstring& element_name, const preconfigured_consumer_factory_t& factory); - spl::shared_ptr - create_consumer(const std::vector& params, - const core::video_format_repository& format_repository, - const std::vector>& channels) const; - spl::shared_ptr - create_consumer(const std::wstring& element_name, - const boost::property_tree::wptree& element, - const core::video_format_repository& format_repository, - const std::vector>& channels) const; + spl::shared_ptr create_consumer(const std::vector& params, + const core::video_format_repository& format_repository, + const std::vector>& channels, + common::bit_depth depth) const; + spl::shared_ptr create_consumer(const std::wstring& element_name, + const boost::property_tree::wptree& element, + const core::video_format_repository& format_repository, + const std::vector>& channels, + common::bit_depth depth) const; private: struct impl; diff --git a/src/modules/artnet/consumer/artnet_consumer.cpp b/src/modules/artnet/consumer/artnet_consumer.cpp index c38b1df6dc..18f7702102 100644 --- a/src/modules/artnet/consumer/artnet_consumer.cpp +++ b/src/modules/artnet/consumer/artnet_consumer.cpp @@ -311,10 +311,14 @@ std::vector get_fixtures_ptree(const boost::property_tree::wptree& ptre spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { configuration config; + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Artnet consumer only supports 8-bit color depth.")); + config.universe = ptree.get(L"universe", config.universe); config.host = ptree.get(L"host", config.host); config.port = ptree.get(L"port", config.port); diff --git a/src/modules/artnet/consumer/artnet_consumer.h b/src/modules/artnet/consumer/artnet_consumer.h index 0fca71248f..e9c0b58172 100644 --- a/src/modules/artnet/consumer/artnet_consumer.h +++ b/src/modules/artnet/consumer/artnet_consumer.h @@ -23,6 +23,7 @@ #include "../util/fixture_calculation.h" +#include #include #include @@ -35,5 +36,6 @@ namespace caspar { namespace artnet { spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::artnet diff --git a/src/modules/bluefish/consumer/bluefish_consumer.cpp b/src/modules/bluefish/consumer/bluefish_consumer.cpp index b5c68d8280..e40cdf7774 100644 --- a/src/modules/bluefish/consumer/bluefish_consumer.cpp +++ b/src/modules/bluefish/consumer/bluefish_consumer.cpp @@ -24,7 +24,6 @@ #include "../util/blue_velvet.h" #include "../util/memory.h" -#include "bluefish_consumer.h" #include #include @@ -884,12 +883,16 @@ struct bluefish_consumer_proxy : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.size() < 1 || !boost::iequals(params.at(0), L"BLUEFISH")) { return core::frame_consumer::empty(); } + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Bluefish consumer only supports 8-bit color depth.")); + configuration config; // const auto device_index = params.size() > 1 ? std::stoi(params.at(1)) : 1; @@ -939,12 +942,16 @@ spl::shared_ptr create_consumer(const std::vector create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { configuration config; auto device_index = ptree.get(L"device", 1); config.device_index = device_index; + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Bluefish consumer only supports 8-bit color depth.")); + auto device_stream = ptree.get(L"sdi-stream", L"1"); if (device_stream == L"1") config.device_stream = bluefish_hardware_output_channel::channel_1; diff --git a/src/modules/bluefish/consumer/bluefish_consumer.h b/src/modules/bluefish/consumer/bluefish_consumer.h index 0d97101bfb..ee32569649 100644 --- a/src/modules/bluefish/consumer/bluefish_consumer.h +++ b/src/modules/bluefish/consumer/bluefish_consumer.h @@ -23,6 +23,7 @@ #pragma once #include +#include #include #include @@ -34,11 +35,13 @@ namespace caspar { namespace bluefish { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::bluefish diff --git a/src/modules/decklink/consumer/decklink_consumer.cpp b/src/modules/decklink/consumer/decklink_consumer.cpp index a16de94557..9d1ab274c9 100644 --- a/src/modules/decklink/consumer/decklink_consumer.cpp +++ b/src/modules/decklink/consumer/decklink_consumer.cpp @@ -902,12 +902,16 @@ struct decklink_consumer_proxy : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.empty() || !boost::iequals(params.at(0), L"DECKLINK")) { return core::frame_consumer::empty(); } + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Decklink consumer only supports 8-bit color depth.")); + configuration config = parse_amcp_config(params, format_repository); return spl::make_shared(config); @@ -916,8 +920,12 @@ spl::shared_ptr create_consumer(const std::vector create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Decklink consumer only supports 8-bit color depth.")); + configuration config = parse_xml_config(ptree, format_repository); return spl::make_shared(config); diff --git a/src/modules/decklink/consumer/decklink_consumer.h b/src/modules/decklink/consumer/decklink_consumer.h index 2d6da8d0bc..7741bf0e04 100644 --- a/src/modules/decklink/consumer/decklink_consumer.h +++ b/src/modules/decklink/consumer/decklink_consumer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -35,10 +36,12 @@ namespace caspar { namespace decklink { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::decklink diff --git a/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp b/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp index 13feda41c3..5d4458c3bc 100644 --- a/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp +++ b/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -715,11 +716,15 @@ struct ffmpeg_consumer : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.size() < 2 || (!boost::iequals(params.at(0), L"STREAM") && !boost::iequals(params.at(0), L"FILE"))) return core::frame_consumer::empty(); + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Ffmpeg consumer only supports 8-bit color depth.")); + auto path = u8(params.at(1)); std::vector args; for (auto n = 2; n < params.size(); ++n) { @@ -731,8 +736,12 @@ spl::shared_ptr create_consumer(const std::vector create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Ffmpeg consumer only supports 8-bit color depth.")); + return spl::make_shared(u8(ptree.get(L"path", L"")), u8(ptree.get(L"args", L"")), ptree.get(L"realtime", false)); diff --git a/src/modules/ffmpeg/consumer/ffmpeg_consumer.h b/src/modules/ffmpeg/consumer/ffmpeg_consumer.h index 3a4af28d6e..b1ef72455a 100644 --- a/src/modules/ffmpeg/consumer/ffmpeg_consumer.h +++ b/src/modules/ffmpeg/consumer/ffmpeg_consumer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -35,10 +36,12 @@ namespace caspar { namespace ffmpeg { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree&, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::ffmpeg diff --git a/src/modules/image/consumer/image_consumer.cpp b/src/modules/image/consumer/image_consumer.cpp index f9c020abff..e60d787932 100644 --- a/src/modules/image/consumer/image_consumer.cpp +++ b/src/modules/image/consumer/image_consumer.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -118,10 +119,14 @@ struct image_consumer : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.empty() || !boost::iequals(params.at(0), L"IMAGE")) return core::frame_consumer::empty(); + + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Image consumer only supports 8-bit color depth.")); std::wstring filename; diff --git a/src/modules/image/consumer/image_consumer.h b/src/modules/image/consumer/image_consumer.h index e971f28e2b..7ccafbfa7a 100644 --- a/src/modules/image/consumer/image_consumer.h +++ b/src/modules/image/consumer/image_consumer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -34,6 +35,7 @@ namespace caspar { namespace image { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::image diff --git a/src/modules/newtek/consumer/newtek_ndi_consumer.cpp b/src/modules/newtek/consumer/newtek_ndi_consumer.cpp index 1a93a2f73e..115c3218c6 100644 --- a/src/modules/newtek/consumer/newtek_ndi_consumer.cpp +++ b/src/modules/newtek/consumer/newtek_ndi_consumer.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -257,10 +258,15 @@ std::atomic newtek_ndi_consumer::instances_(0); spl::shared_ptr create_ndi_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.size() < 1 || !boost::iequals(params.at(0), L"NDI")) return core::frame_consumer::empty(); + + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Newtek NDI consumer only supports 8-bit color depth.")); + std::wstring name = get_param(L"NAME", params, L""); bool allow_fields = contains_param(L"ALLOW_FIELDS", params); return spl::make_shared(name, allow_fields); @@ -269,10 +275,15 @@ create_ndi_consumer(const std::vector& par spl::shared_ptr create_preconfigured_ndi_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { auto name = ptree.get(L"name", L""); bool allow_fields = ptree.get(L"allow-fields", false); + + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Newtek NDI consumer only supports 8-bit color depth.")); + return spl::make_shared(name, allow_fields); } diff --git a/src/modules/newtek/consumer/newtek_ndi_consumer.h b/src/modules/newtek/consumer/newtek_ndi_consumer.h index 2f3e788d53..e5081d05c2 100644 --- a/src/modules/newtek/consumer/newtek_ndi_consumer.h +++ b/src/modules/newtek/consumer/newtek_ndi_consumer.h @@ -22,6 +22,7 @@ #pragma once +#include #include #include @@ -35,10 +36,12 @@ namespace caspar { namespace newtek { spl::shared_ptr create_ndi_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_ndi_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::newtek diff --git a/src/modules/oal/consumer/oal_consumer.cpp b/src/modules/oal/consumer/oal_consumer.cpp index fa790fe9fb..5484c75ec0 100644 --- a/src/modules/oal/consumer/oal_consumer.cpp +++ b/src/modules/oal/consumer/oal_consumer.cpp @@ -389,7 +389,8 @@ struct oal_consumer : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.empty() || !boost::iequals(params.at(0), L"AUDIO")) return core::frame_consumer::empty(); @@ -400,7 +401,8 @@ spl::shared_ptr create_consumer(const std::vector create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { return spl::make_shared(); } diff --git a/src/modules/oal/consumer/oal_consumer.h b/src/modules/oal/consumer/oal_consumer.h index a868d505b6..950c9c7ba1 100644 --- a/src/modules/oal/consumer/oal_consumer.h +++ b/src/modules/oal/consumer/oal_consumer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -34,10 +35,12 @@ namespace caspar { namespace oal { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree&, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::oal diff --git a/src/modules/screen/consumer/screen_consumer.cpp b/src/modules/screen/consumer/screen_consumer.cpp index 23d4301d58..3080b70650 100644 --- a/src/modules/screen/consumer/screen_consumer.cpp +++ b/src/modules/screen/consumer/screen_consumer.cpp @@ -608,7 +608,8 @@ struct screen_consumer_proxy : public core::frame_consumer spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { if (params.empty() || !boost::iequals(params.at(0), L"SCREEN")) { return core::frame_consumer::empty(); @@ -616,6 +617,9 @@ spl::shared_ptr create_consumer(const std::vector 1) { try { config.screen_index = std::stoi(params.at(1)); @@ -644,9 +648,14 @@ spl::shared_ptr create_consumer(const std::vector create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels) + const std::vector>& channels, + common::bit_depth depth) { configuration config; + + if (depth != common::bit_depth::bit8) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Screen consumer only supports 8-bit color depth.")); + config.name = ptree.get(L"name", config.name); config.screen_index = ptree.get(L"device", config.screen_index + 1) - 1; config.screen_x = ptree.get(L"x", config.screen_x); diff --git a/src/modules/screen/consumer/screen_consumer.h b/src/modules/screen/consumer/screen_consumer.h index c7129052b4..69365d5c05 100644 --- a/src/modules/screen/consumer/screen_consumer.h +++ b/src/modules/screen/consumer/screen_consumer.h @@ -21,6 +21,7 @@ #pragma once +#include #include #include @@ -33,10 +34,12 @@ namespace caspar { namespace screen { spl::shared_ptr create_consumer(const std::vector& params, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); spl::shared_ptr create_preconfigured_consumer(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository, - const std::vector>& channels); + const std::vector>& channels, + common::bit_depth depth); }} // namespace caspar::screen diff --git a/src/protocol/amcp/AMCPCommandsImpl.cpp b/src/protocol/amcp/AMCPCommandsImpl.cpp index 0a8c812eca..a60b9fe779 100644 --- a/src/protocol/amcp/AMCPCommandsImpl.cpp +++ b/src/protocol/amcp/AMCPCommandsImpl.cpp @@ -478,8 +478,10 @@ std::wstring add_command(command_context& ctx) core::diagnostics::scoped_call_context save; core::diagnostics::call_context::for_thread().video_channel = ctx.channel_index + 1; - auto consumer = ctx.static_context->consumer_registry->create_consumer( - ctx.parameters, ctx.static_context->format_repository, get_channels(ctx)); + auto consumer = ctx.static_context->consumer_registry->create_consumer(ctx.parameters, + ctx.static_context->format_repository, + get_channels(ctx), + ctx.channel.raw_channel->mixer().depth()); ctx.channel.raw_channel->output().add(ctx.layer_index(consumer->index()), consumer); return L"202 ADD OK\r\n"; @@ -497,7 +499,10 @@ std::wstring remove_command(command_context& ctx) } index = ctx.static_context->consumer_registry - ->create_consumer(ctx.parameters, ctx.static_context->format_repository, get_channels(ctx)) + ->create_consumer(ctx.parameters, + ctx.static_context->format_repository, + get_channels(ctx), + ctx.channel.raw_channel->mixer().depth()) ->index(); } @@ -510,8 +515,11 @@ std::wstring remove_command(command_context& ctx) std::wstring print_command(command_context& ctx) { - ctx.channel.raw_channel->output().add(ctx.static_context->consumer_registry->create_consumer( - {L"IMAGE"}, ctx.static_context->format_repository, get_channels(ctx))); + ctx.channel.raw_channel->output().add( + ctx.static_context->consumer_registry->create_consumer({L"IMAGE"}, + ctx.static_context->format_repository, + get_channels(ctx), + ctx.channel.raw_channel->mixer().depth())); return L"202 PRINT OK\r\n"; } @@ -1377,7 +1385,7 @@ std::wstring channel_grid_command(command_context& ctx) params.emplace_back(L"NAME"); params.emplace_back(L"Channel Grid Window"); auto screen = ctx.static_context->consumer_registry->create_consumer( - params, ctx.static_context->format_repository, get_channels(ctx)); + params, ctx.static_context->format_repository, get_channels(ctx), ctx.channel.raw_channel->mixer().depth()); self.raw_channel->output().add(screen); diff --git a/src/shell/server.cpp b/src/shell/server.cpp index d5749b0961..b8c5fc7b92 100644 --- a/src/shell/server.cpp +++ b/src/shell/server.cpp @@ -341,8 +341,12 @@ struct server::impl try { if (name != L"") - channel.raw_channel->output().add(consumer_registry_->create_consumer( - name, xml_consumer.second, video_format_repository_, channels_vec)); + channel.raw_channel->output().add( + consumer_registry_->create_consumer(name, + xml_consumer.second, + video_format_repository_, + channels_vec, + channel.raw_channel->mixer().depth())); } catch (...) { CASPAR_LOG_CURRENT_EXCEPTION(); } From e4e0bcc52613ef8bd7f4feef5af7f0b1a9329f30 Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 29 Mar 2024 19:19:34 +0000 Subject: [PATCH 03/21] Expose color depth in config --- src/shell/casparcg.config | 1 + src/shell/server.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/shell/casparcg.config b/src/shell/casparcg.config index b1c47e36a7..f35e79e07a 100644 --- a/src/shell/casparcg.config +++ b/src/shell/casparcg.config @@ -79,6 +79,7 @@ PAL [PAL|NTSC|576p2500|720p2398|720p2400|720p2500|720p5000|720p2997|720p5994|720p3000|720p6000|1080p2398|1080p2400|1080i5000|1080i5994|1080i6000|1080p2500|1080p2997|1080p3000|1080p5000|1080p5994|1080p6000|1556p2398|1556p2400|1556p2500|dci1080p2398|dci1080p2400|dci1080p2500|2160p2398|2160p2400|2160p2500|2160p2997|2160p3000|2160p5000|2160p5994|2160p6000|dci2160p2398|dci2160p2400|dci2160p2500] + 8 [8|16] [1..] diff --git a/src/shell/server.cpp b/src/shell/server.cpp index b8c5fc7b92..2116891f34 100644 --- a/src/shell/server.cpp +++ b/src/shell/server.cpp @@ -256,15 +256,21 @@ struct server::impl auto format_desc_str = xml_channel.second.get(L"video-mode", L"PAL"); auto format_desc = video_format_repository_.find(format_desc_str); + auto color_depth = xml_channel.second.get(L"color-depth", 8); + if (color_depth != 8 && color_depth != 16) + CASPAR_THROW_EXCEPTION(user_error() + << msg_info(L"Invalid color-depth: " + std::to_wstring(color_depth))); + if (format_desc.format == video_format::invalid) CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Invalid video-mode: " + format_desc_str)); auto weak_client = std::weak_ptr(osc_client_); auto channel_id = static_cast(channels_->size() + 1); + auto depth = color_depth == 16 ? common::bit_depth::bit16 : common::bit_depth::bit8; auto channel = spl::make_shared(channel_id, format_desc, - accelerator_.create_image_mixer(channel_id, common::bit_depth::bit8), + accelerator_.create_image_mixer(channel_id, depth), [channel_id, weak_client](core::monitor::state channel_state) { monitor::state state; state[""]["channel"][channel_id] = channel_state; From 9a0f8e8333dd2c4813739e56b3625ead39ed4adc Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Sun, 31 Mar 2024 20:33:21 +0000 Subject: [PATCH 04/21] Add support for color depth in ffmpeg consumer --- .../ffmpeg/consumer/ffmpeg_consumer.cpp | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp b/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp index 5d4458c3bc..c2efcdfc53 100644 --- a/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp +++ b/src/modules/ffmpeg/consumer/ffmpeg_consumer.cpp @@ -24,9 +24,9 @@ #include "../util/av_assert.h" #include "../util/av_util.h" +#include #include #include -#include #include #include #include @@ -100,6 +100,7 @@ struct Stream AVCodecID codec_id, const core::video_format_desc& format_desc, bool realtime, + common::bit_depth depth, std::map& options) { std::map stream_options; @@ -178,8 +179,10 @@ struct Stream const auto sar = boost::rational(format_desc.square_width, format_desc.square_height) / boost::rational(format_desc.width, format_desc.height); + const auto pix_fmt = (depth == common::bit_depth::bit8) ? AV_PIX_FMT_YUVA422P : AV_PIX_FMT_YUVA422P10; + auto args = (boost::format("video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:sar=%d/%d:frame_rate=%d/%d") % - format_desc.width % format_desc.height % AV_PIX_FMT_YUVA422P % format_desc.duration % + format_desc.width % format_desc.height % pix_fmt % format_desc.duration % (format_desc.time_scale * format_desc.field_count) % sar.numerator() % sar.denominator() % (format_desc.framerate.numerator() * format_desc.field_count) % format_desc.framerate.denominator()) @@ -464,8 +467,10 @@ struct ffmpeg_consumer : public core::frame_consumer tbb::concurrent_bounded_queue frame_buffer_; std::thread frame_thread_; + common::bit_depth depth_; + public: - ffmpeg_consumer(std::string path, std::string args, bool realtime) + ffmpeg_consumer(std::string path, std::string args, bool realtime, common::bit_depth depth) : channel_index_([&] { boost::crc_16_type result; result.process_bytes(path.data(), path.length()); @@ -474,6 +479,7 @@ struct ffmpeg_consumer : public core::frame_consumer , realtime_(realtime) , path_(std::move(path)) , args_(std::move(args)) + , depth_(depth) { state_["file/path"] = u8(path_); @@ -558,7 +564,7 @@ struct ffmpeg_consumer : public core::frame_consumer if (oc->oformat->video_codec == AV_CODEC_ID_H264 && options.find("preset:v") == options.end()) { options["preset:v"] = "veryfast"; } - video_stream.emplace(oc, ":v", oc->oformat->video_codec, format_desc, realtime_, options); + video_stream.emplace(oc, ":v", oc->oformat->video_codec, format_desc, realtime_, depth_, options); { std::lock_guard lock(state_mutex_); @@ -568,7 +574,7 @@ struct ffmpeg_consumer : public core::frame_consumer std::optional audio_stream; if (oc->oformat->audio_codec != AV_CODEC_ID_NONE) { - audio_stream.emplace(oc, ":a", oc->oformat->audio_codec, format_desc, realtime_, options); + audio_stream.emplace(oc, ":a", oc->oformat->audio_codec, format_desc, realtime_, depth_, options); } if (!(oc->oformat->flags & AVFMT_NOFILE)) { @@ -722,15 +728,13 @@ spl::shared_ptr create_consumer(const std::vector args; for (auto n = 2; n < params.size(); ++n) { args.emplace_back(u8(params[n])); } - return spl::make_shared(path, boost::join(args, " "), boost::iequals(params.at(0), L"STREAM")); + return spl::make_shared( + path, boost::join(args, " "), boost::iequals(params.at(0), L"STREAM"), depth); } spl::shared_ptr @@ -739,11 +743,9 @@ create_preconfigured_consumer(const boost::property_tree::wptree& const std::vector>& channels, common::bit_depth depth) { - if (depth != common::bit_depth::bit8) - CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Ffmpeg consumer only supports 8-bit color depth.")); - return spl::make_shared(u8(ptree.get(L"path", L"")), u8(ptree.get(L"args", L"")), - ptree.get(L"realtime", false)); + ptree.get(L"realtime", false), + depth); } }} // namespace caspar::ffmpeg From e3289ecac25fca89221f258051222ecdefa08d92 Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:40:50 +0000 Subject: [PATCH 05/21] Add support for higher color depths in ffmpeg producer --- src/modules/ffmpeg/producer/av_producer.cpp | 4 + src/modules/ffmpeg/util/av_util.cpp | 104 +++++++++++--------- src/modules/ffmpeg/util/av_util.h | 1 - 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/src/modules/ffmpeg/producer/av_producer.cpp b/src/modules/ffmpeg/producer/av_producer.cpp index e8c4a0f6a6..f18595bf9e 100644 --- a/src/modules/ffmpeg/producer/av_producer.cpp +++ b/src/modules/ffmpeg/producer/av_producer.cpp @@ -502,7 +502,11 @@ struct Filter AV_PIX_FMT_ABGR, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV422P, + AV_PIX_FMT_YUV422P10, + AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P, + AV_PIX_FMT_YUV420P10, + AV_PIX_FMT_YUV420P12, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P, diff --git a/src/modules/ffmpeg/util/av_util.cpp b/src/modules/ffmpeg/util/av_util.cpp index fa6a2e4736..b853f5b7db 100644 --- a/src/modules/ffmpeg/util/av_util.cpp +++ b/src/modules/ffmpeg/util/av_util.cpp @@ -1,7 +1,8 @@ #include "av_util.h" - #include "av_assert.h" +#include + #if defined(_MSC_VER) #pragma warning(push) #pragma warning(disable : 4244) @@ -22,6 +23,8 @@ extern "C" { #include #include +#include + namespace caspar { namespace ffmpeg { std::shared_ptr alloc_frame() @@ -93,43 +96,51 @@ core::mutable_frame make_frame(void* tag, return frame; } -core::pixel_format get_pixel_format(AVPixelFormat pix_fmt) +std::tuple get_pixel_format(AVPixelFormat pix_fmt) { switch (pix_fmt) { case AV_PIX_FMT_GRAY8: - return core::pixel_format::gray; + return {core::pixel_format::gray, common::bit_depth::bit8}; case AV_PIX_FMT_RGB24: - return core::pixel_format::rgb; + return {core::pixel_format::rgb, common::bit_depth::bit8}; case AV_PIX_FMT_BGR24: - return core::pixel_format::bgr; + return {core::pixel_format::bgr, common::bit_depth::bit8}; case AV_PIX_FMT_BGRA: - return core::pixel_format::bgra; + return {core::pixel_format::bgra, common::bit_depth::bit8}; case AV_PIX_FMT_ARGB: - return core::pixel_format::argb; + return {core::pixel_format::argb, common::bit_depth::bit8}; case AV_PIX_FMT_RGBA: - return core::pixel_format::rgba; + return {core::pixel_format::rgba, common::bit_depth::bit8}; case AV_PIX_FMT_ABGR: - return core::pixel_format::abgr; + return {core::pixel_format::abgr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV444P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV422P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; + case AV_PIX_FMT_YUV422P10: + return {core::pixel_format::ycbcr, common::bit_depth::bit10}; + case AV_PIX_FMT_YUV422P12: + return {core::pixel_format::ycbcr, common::bit_depth::bit12}; case AV_PIX_FMT_YUV420P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; + case AV_PIX_FMT_YUV420P10: + return {core::pixel_format::ycbcr, common::bit_depth::bit10}; + case AV_PIX_FMT_YUV420P12: + return {core::pixel_format::ycbcr, common::bit_depth::bit12}; case AV_PIX_FMT_YUV411P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV410P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA420P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA422P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA444P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_UYVY422: - return core::pixel_format::uyvy; + return {core::pixel_format::uyvy, common::bit_depth::bit8}; default: - return core::pixel_format::invalid; + return {core::pixel_format::invalid, common::bit_depth::bit8}; } } @@ -139,24 +150,26 @@ core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int int linesizes[4]; av_image_fill_linesizes(linesizes, pix_fmt, width); - core::pixel_format_desc desc = core::pixel_format_desc(get_pixel_format(pix_fmt)); + const auto fmt = get_pixel_format(pix_fmt); + auto desc = core::pixel_format_desc(std::get<0>(fmt)); + auto depth = std::get<1>(fmt); switch (desc.format) { case core::pixel_format::gray: case core::pixel_format::luma: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0], height, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); return desc; } case core::pixel_format::bgr: case core::pixel_format::rgb: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 3, height, 3)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 3, height, 3, depth)); return desc; } case core::pixel_format::bgra: case core::pixel_format::argb: case core::pixel_format::rgba: case core::pixel_format::abgr: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 4, height, 4)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 4, height, 4, depth)); return desc; } case core::pixel_format::ycbcr: @@ -177,20 +190,22 @@ core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int av_image_fill_pointers(dummy_pict_data, pix_fmt, height, NULL, linesizes); auto size2 = static_cast(dummy_pict_data[2] - dummy_pict_data[1]); #endif - auto h2 = size2 / linesizes[1]; + auto h2 = size2 / linesizes[1]; + auto factor1 = linesizes[0] / linesizes[1]; + auto factor2 = linesizes[0] / linesizes[2]; - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0], height, 1)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[1], h2, 1)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[2], h2, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / factor1, h2, 1, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / factor2, h2, 1, depth)); if (desc.format == core::pixel_format::ycbcra) - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[3], height, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); return desc; } case core::pixel_format::uyvy: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 2, height, 2)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 4, height, 4)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 2, height, 2, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 4, height, 4, depth)); data_map.clear(); data_map.push_back(0); @@ -220,28 +235,29 @@ std::shared_ptr make_av_video_frame(const core::const_frame& frame, con av_frame->width = format_desc.width; av_frame->height = format_desc.height; + const auto is_16bit = planes[0].depth != common::bit_depth::bit8; switch (format) { case core::pixel_format::rgb: - av_frame->format = AVPixelFormat::AV_PIX_FMT_RGB24; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_RGB48 : AVPixelFormat::AV_PIX_FMT_RGB24; break; case core::pixel_format::bgr: - av_frame->format = AVPixelFormat::AV_PIX_FMT_BGR24; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGR48 : AVPixelFormat::AV_PIX_FMT_BGR24; break; case core::pixel_format::rgba: - av_frame->format = AVPixelFormat::AV_PIX_FMT_RGBA; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_RGBA64 : AVPixelFormat::AV_PIX_FMT_RGBA; break; case core::pixel_format::argb: - av_frame->format = AVPixelFormat::AV_PIX_FMT_ARGB; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_ARGB; break; case core::pixel_format::bgra: - av_frame->format = AVPixelFormat::AV_PIX_FMT_BGRA; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_BGRA; break; case core::pixel_format::abgr: - av_frame->format = AVPixelFormat::AV_PIX_FMT_ABGR; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_ABGR; break; case core::pixel_format::gray: case core::pixel_format::luma: - av_frame->format = AVPixelFormat::AV_PIX_FMT_GRAY8; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_GRAY16 : AVPixelFormat::AV_PIX_FMT_GRAY8; break; case core::pixel_format::ycbcr: { int y_w = planes[0].width; @@ -250,27 +266,27 @@ std::shared_ptr make_av_video_frame(const core::const_frame& frame, con int c_h = planes[1].height; if (c_h == y_h && c_w == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV444P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV444P10 : AVPixelFormat::AV_PIX_FMT_YUV444P; else if (c_h == y_h && c_w * 2 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV422P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV422P10 : AVPixelFormat::AV_PIX_FMT_YUV422P; else if (c_h == y_h && c_w * 4 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV411P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV422P10 : AVPixelFormat::AV_PIX_FMT_YUV411P; else if (c_h * 2 == y_h && c_w * 2 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV420P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV420P10 : AVPixelFormat::AV_PIX_FMT_YUV420P; else if (c_h * 2 == y_h && c_w * 4 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV410P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV420P10 : AVPixelFormat::AV_PIX_FMT_YUV410P; break; } case core::pixel_format::ycbcra: - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUVA420P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUVA420P10 : AVPixelFormat::AV_PIX_FMT_YUVA420P; break; case core::pixel_format::count: case core::pixel_format::invalid: break; } - FF(av_frame_get_buffer(av_frame.get(), 32)); + FF(av_frame_get_buffer(av_frame.get(), is_16bit ? 64 : 32)); // TODO (perf) Avoid extra memcpy. for (int n = 0; n < planes.size(); ++n) { diff --git a/src/modules/ffmpeg/util/av_util.h b/src/modules/ffmpeg/util/av_util.h index 8a6ceed869..c7e5f2ff62 100644 --- a/src/modules/ffmpeg/util/av_util.h +++ b/src/modules/ffmpeg/util/av_util.h @@ -20,7 +20,6 @@ namespace caspar { namespace ffmpeg { std::shared_ptr alloc_frame(); std::shared_ptr alloc_packet(); -core::pixel_format get_pixel_format(AVPixelFormat pix_fmt); core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int height, std::vector& data_map); core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory, From e276ffb84f38ccb702c83174a7fddba325761c49 Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:15:43 +0200 Subject: [PATCH 06/21] Add support for higher color depth in decklink consumer --- src/common/memshfl.h | 8 +- src/modules/decklink/consumer/config.h | 3 +- .../decklink/consumer/decklink_consumer.cpp | 260 ++++++++++++++++-- src/modules/decklink/consumer/frame.cpp | 64 ++++- src/modules/decklink/consumer/frame.h | 10 +- 5 files changed, 301 insertions(+), 44 deletions(-) diff --git a/src/common/memshfl.h b/src/common/memshfl.h index 3947e9867f..26e129ede9 100644 --- a/src/common/memshfl.h +++ b/src/common/memshfl.h @@ -35,14 +35,14 @@ namespace caspar { #ifdef _MSC_VER -static std::shared_ptr create_aligned_buffer(size_t size) +static std::shared_ptr create_aligned_buffer(size_t size, size_t alignment = 64) { - return std::shared_ptr(_aligned_malloc(size, 64), _aligned_free); + return std::shared_ptr(_aligned_malloc(size, alignment), _aligned_free); } #else -static std::shared_ptr create_aligned_buffer(size_t size) +static std::shared_ptr create_aligned_buffer(size_t size, size_t alignment = 64) { - return std::shared_ptr(aligned_alloc(64, size), free); + return std::shared_ptr(aligned_alloc(alignment, size), free); } #endif diff --git a/src/modules/decklink/consumer/config.h b/src/modules/decklink/consumer/config.h index b10dfc06fb..a5ed339044 100644 --- a/src/modules/decklink/consumer/config.h +++ b/src/modules/decklink/consumer/config.h @@ -82,6 +82,7 @@ struct configuration wait_for_reference_t wait_for_reference = wait_for_reference_t::automatic; int wait_for_reference_duration = 10; // seconds int base_buffer_depth = 3; + bool hdr = false; port_configuration primary; std::vector secondaries; @@ -101,4 +102,4 @@ configuration parse_xml_config(const boost::property_tree::wptree& ptree, configuration parse_amcp_config(const std::vector& params, const core::video_format_repository& format_repository); -}} // namespace caspar::decklink \ No newline at end of file +}} // namespace caspar::decklink diff --git a/src/modules/decklink/consumer/decklink_consumer.cpp b/src/modules/decklink/consumer/decklink_consumer.cpp index 9d1ab274c9..00c0c5b265 100644 --- a/src/modules/decklink/consumer/decklink_consumer.cpp +++ b/src/modules/decklink/consumer/decklink_consumer.cpp @@ -77,7 +77,8 @@ void set_latency(const com_iface_ptr& config, com_ptr get_display_mode(const com_iface_ptr& device, core::video_format fmt, BMDPixelFormat pix_fmt, - BMDSupportedVideoModeFlags flag) + BMDSupportedVideoModeFlags flag, + bool hdr) { auto format = get_decklink_video_format(fmt); @@ -99,14 +100,15 @@ com_ptr get_display_mode(const com_iface_ptrGetDisplayMode(); if (FAILED(device->DoesSupportVideoMode( - bmdVideoConnectionUnspecified, mode->GetDisplayMode(), pix_fmt, flag, &actualMode, &supported))) + bmdVideoConnectionUnspecified, displayMode, pix_fmt, flag, &actualMode, &supported))) CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(L"Could not determine whether device supports requested video format: " + get_mode_name(mode))); else if (!supported) CASPAR_LOG(info) << L"Device may not support video-format: " << get_mode_name(mode); - else if (actualMode != bmdModeUnknown) + else if (actualMode != bmdModeUnknown && actualMode != displayMode) CASPAR_LOG(warning) << L"Device supports video-format with conversion: " << get_mode_name(mode); return mode; @@ -191,24 +193,93 @@ core::video_format_desc get_decklink_format(const port_configuration& confi return fallback_format_desc; } -class decklink_frame : public IDeckLinkVideoFrame +enum EOTF +{ + SDR = 0, + HDR = 1, + PQ = 2, + HLG = 3 +}; + +struct ChromaticityCoordinates +{ + double RedX; + double RedY; + double GreenX; + double GreenY; + double BlueX; + double BlueY; + double WhiteX; + double WhiteY; +}; + +struct HDRMetadata +{ + int64_t EOTF; + double maxDisplayMasteringLuminance; + double minDisplayMasteringLuminance; + double maxCLL; + double maxFALL; +}; + +const auto REC_709 = ChromaticityCoordinates{0.640, 0.330, 0.300, 0.600, 0.150, 0.060, 0.3127, 0.3290}; +const auto REC_2020 = ChromaticityCoordinates{0.708, 0.292, 0.170, 0.797, 0.131, 0.046, 0.3127, 0.3290}; + +class decklink_frame + : public IDeckLinkVideoFrame + , public IDeckLinkVideoFrameMetadataExtensions { core::video_format_desc format_desc_; std::shared_ptr data_; std::atomic ref_count_{0}; int nb_samples_; + const bool hdr_; + BMDFrameFlags flags_; + BMDPixelFormat pix_fmt_; public: - decklink_frame(std::shared_ptr data, core::video_format_desc format_desc, int nb_samples) + decklink_frame(std::shared_ptr data, core::video_format_desc format_desc, int nb_samples, bool hdr) : format_desc_(std::move(format_desc)) , data_(std::move(data)) , nb_samples_(nb_samples) + , hdr_(hdr) + , flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault) + , pix_fmt_(get_pixel_format(hdr)) { } // IUnknown - HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*) override { return E_NOINTERFACE; } + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv) override + { + /* Implementation from the SignalGenHDR example in the Decklink SDK */ + +#ifdef _WIN32 + IID iunknown = IID_IUnknown; +#else + REFIID iunknown = IID_IUnknown; +#endif + HRESULT result = E_NOINTERFACE; + + if (ppv == nullptr) + return E_INVALIDARG; + + //// Initialise the return result + *ppv = nullptr; + + if (std::memcmp(&iid, &iunknown, sizeof(REFIID)) == 0) { + *ppv = this; + AddRef(); + } else if (std::memcmp(&iid, &IID_IDeckLinkVideoFrame, sizeof(REFIID)) == 0) { + *ppv = static_cast(this); + AddRef(); + } else if (hdr_ && std::memcmp(&iid, &IID_IDeckLinkVideoFrameMetadataExtensions, sizeof(REFIID)) == 0) { + *ppv = static_cast(this); + AddRef(); + } + + return result; + } ULONG STDMETHODCALLTYPE AddRef() override { return ++ref_count_; } @@ -227,9 +298,9 @@ class decklink_frame : public IDeckLinkVideoFrame long STDMETHODCALLTYPE GetWidth() override { return static_cast(format_desc_.width); } long STDMETHODCALLTYPE GetHeight() override { return static_cast(format_desc_.height); } - long STDMETHODCALLTYPE GetRowBytes() override { return static_cast(format_desc_.width) * 4; } - BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override { return bmdFormat8BitBGRA; } - BMDFrameFlags STDMETHODCALLTYPE GetFlags() override { return bmdFrameFlagDefault; } + long STDMETHODCALLTYPE GetRowBytes() override { return static_cast(get_row_bytes(format_desc_, hdr_)); } + BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override { return pix_fmt_; } + BMDFrameFlags STDMETHODCALLTYPE GetFlags() override { return flags_; } HRESULT STDMETHODCALLTYPE GetBytes(void** buffer) override { @@ -245,6 +316,109 @@ class decklink_frame : public IDeckLinkVideoFrame HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary) override { return S_FALSE; } [[nodiscard]] int nb_samples() const { return nb_samples_; } + + // IDeckLinkVideoFrameMetadataExtensions + HRESULT STDMETHODCALLTYPE GetInt(BMDDeckLinkFrameMetadataID metadataID, int64_t* value) + { + HRESULT result = S_OK; + + switch (metadataID) { + case bmdDeckLinkFrameMetadataHDRElectroOpticalTransferFunc: + *value = EOTF::PQ; + break; + + case bmdDeckLinkFrameMetadataColorspace: + *value = bmdColorspaceRec709; + break; + + default: + value = nullptr; + result = E_INVALIDARG; + } + + return result; + } + + HRESULT STDMETHODCALLTYPE GetFloat(BMDDeckLinkFrameMetadataID metadataID, double* value) + { + HRESULT result = S_OK; + + switch (metadataID) { + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedX: + *value = REC_709.RedX; + break; + + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedY: + *value = REC_709.RedY; + break; + + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenX: + *value = REC_709.GreenX; + break; + + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenY: + *value = REC_709.GreenY; + break; + + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueX: + *value = REC_709.BlueX; + break; + + case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueY: + *value = REC_709.BlueY; + break; + + case bmdDeckLinkFrameMetadataHDRWhitePointX: + *value = REC_709.WhiteX; + break; + + case bmdDeckLinkFrameMetadataHDRWhitePointY: + *value = REC_709.WhiteY; + break; + + case bmdDeckLinkFrameMetadataHDRMaxDisplayMasteringLuminance: + *value = 1000.0; + break; + + case bmdDeckLinkFrameMetadataHDRMinDisplayMasteringLuminance: + *value = 0.005; + break; + + case bmdDeckLinkFrameMetadataHDRMaximumContentLightLevel: + *value = 1000.0; + break; + + case bmdDeckLinkFrameMetadataHDRMaximumFrameAverageLightLevel: + *value = 50.0; + break; + + default: + value = nullptr; + result = E_INVALIDARG; + } + + return result; + } + + HRESULT STDMETHODCALLTYPE GetFlag(BMDDeckLinkFrameMetadataID, BOOL* value) + { + // Not expecting GetFlag + *value = false; + return E_INVALIDARG; + } + + HRESULT STDMETHODCALLTYPE GetString(BMDDeckLinkFrameMetadataID, String* value) + { + // Not expecting GetString + *value = nullptr; + return E_INVALIDARG; + } + + HRESULT STDMETHODCALLTYPE GetBytes(BMDDeckLinkFrameMetadataID metadataID, void* buffer, uint32_t* bufferSize) + { + *bufferSize = 0; + return E_INVALIDARG; + } }; struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback @@ -265,8 +439,11 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback const core::video_format_desc channel_format_desc_; const core::video_format_desc decklink_format_desc_; - com_ptr mode_ = - get_display_mode(output_, decklink_format_desc_.format, bmdFormat8BitBGRA, bmdSupportedVideoModeDefault); + com_ptr mode_ = get_display_mode(output_, + decklink_format_desc_.format, + get_pixel_format(config_.hdr), + bmdSupportedVideoModeDefault, + config_.hdr); decklink_secondary_port(const configuration& config, port_configuration output_config, @@ -367,8 +544,13 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback frame1 = frame; } - auto image_data = convert_frame_for_port( - channel_format_desc_, decklink_format_desc_, output_config_, frame1, frame2, mode_->GetFieldDominance()); + auto image_data = convert_frame_for_port(channel_format_desc_, + decklink_format_desc_, + output_config_, + frame1, + frame2, + mode_->GetFieldDominance(), + config_.hdr); schedule_next_video(image_data, 0, display_time); } @@ -376,7 +558,7 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback void schedule_next_video(std::shared_ptr image_data, int nb_samples, BMDTimeValue display_time) { auto packed_frame = wrap_raw( - new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples)); + new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr)); if (FAILED(output_->ScheduleVideoFrame(get_raw(packed_frame), display_time, decklink_format_desc_.duration, @@ -439,8 +621,11 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback std::vector> secondary_port_contexts_; int device_sync_group_ = 0; - com_ptr mode_ = - get_display_mode(output_, decklink_format_desc_.format, bmdFormat8BitBGRA, bmdSupportedVideoModeDefault); + com_ptr mode_ = get_display_mode(output_, + decklink_format_desc_.format, + get_pixel_format(config_.hdr), + bmdSupportedVideoModeDefault, + config_.hdr); std::atomic abort_request_{false}; @@ -506,6 +691,14 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback set_latency(configuration_, config.latency, print()); set_keyer(attributes_, keyer_, config.keyer, print()); + if (config.hdr) { + BOOL flag = FALSE; + if (SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsHDRMetadata, &flag)) && !flag) + CASPAR_LOG(error) << print() << L" Device does not support HDR metadata."; + if (SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsColorspaceMetadata, &flag)) && !flag) + CASPAR_LOG(warning) << print() << L" Device does not support colorspace metadata."; + } + if (config.embedded_audio) { output_->BeginAudioPreroll(); } @@ -518,7 +711,7 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback nb_samples); } - std::shared_ptr image_data = create_aligned_buffer(decklink_format_desc_.size); + std::shared_ptr image_data = allocate_frame_data(decklink_format_desc_, config_.hdr); schedule_next_video(image_data, nb_samples, video_scheduled_); for (auto& context : secondary_port_contexts_) { @@ -739,7 +932,8 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback config_.primary, frame1, frame2, - mode_->GetFieldDominance()); + mode_->GetFieldDominance(), + config_.hdr); schedule_next_video(image_data, nb_samples, video_display_time); @@ -804,7 +998,7 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback void schedule_next_video(std::shared_ptr image_data, int nb_samples, BMDTimeValue display_time) { auto fill_frame = wrap_raw( - new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples)); + new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr)); if (FAILED(output_->ScheduleVideoFrame( get_raw(fill_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) { CASPAR_LOG(error) << print() << L" Failed to schedule primary video."; @@ -909,11 +1103,15 @@ spl::shared_ptr create_consumer(const std::vector(config); } @@ -923,11 +1121,23 @@ create_preconfigured_consumer(const boost::property_tree::wptree& const std::vector>& channels, common::bit_depth depth) { - if (depth != common::bit_depth::bit8) - CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Decklink consumer only supports 8-bit color depth.")); - configuration config = parse_xml_config(ptree, format_repository); + config.hdr = (depth != common::bit_depth::bit8); + + if (config.hdr && config.primary.has_subregion_geometry()) { + CASPAR_THROW_EXCEPTION(caspar_exception() + << msg_info("Decklink consumer does not support hdr in combination with sub regions.")); + } + if (config.hdr && config.secondaries.size() > 0) { + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info( + "Decklink consumer does not support hdr in combination with secondary ports.")); + } + if (config.hdr && config.primary.key_only) { + CASPAR_THROW_EXCEPTION(caspar_exception() + << msg_info("Decklink consumer does not support hdr in combination with key only")); + } + return spl::make_shared(config); } diff --git a/src/modules/decklink/consumer/frame.cpp b/src/modules/decklink/consumer/frame.cpp index 57d55e6640..f3e3f77570 100644 --- a/src/modules/decklink/consumer/frame.cpp +++ b/src/modules/decklink/consumer/frame.cpp @@ -25,10 +25,24 @@ #include +#include #include namespace caspar { namespace decklink { +BMDPixelFormat get_pixel_format(bool hdr) { return hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA; } +int get_row_bytes(const core::video_format_desc& format_desc, bool hdr) +{ + return hdr ? ((format_desc.width + 63) / 64) * 256 : format_desc.width * 4; +} + +std::shared_ptr allocate_frame_data(const core::video_format_desc& format_desc, bool hdr) +{ + auto alignment = hdr ? 256 : 64; + auto size = hdr ? get_row_bytes(format_desc, hdr) * format_desc.height : format_desc.size; + return create_aligned_buffer(size, alignment); +} + std::shared_ptr convert_to_key_only(const std::shared_ptr& image_data, std::size_t byte_count) { auto key_data = create_aligned_buffer(byte_count); @@ -43,7 +57,8 @@ void convert_frame(const core::video_format_desc& channel_format_desc, const port_configuration& config, std::shared_ptr& image_data, bool topField, - const core::const_frame& frame) + const core::const_frame& frame, + bool hdr) { // No point copying an empty frame if (!frame) @@ -54,11 +69,33 @@ void convert_frame(const core::video_format_desc& channel_format_desc, if (channel_format_desc.format == decklink_format_desc.format && config.src_x == 0 && config.src_y == 0 && config.region_w == 0 && config.region_h == 0 && config.dest_x == 0 && config.dest_y == 0) { // Fast path - size_t byte_count_line = (size_t)decklink_format_desc.width * 4; - for (int y = firstLine; y < decklink_format_desc.height; y += decklink_format_desc.field_count) { - std::memcpy(reinterpret_cast(image_data.get()) + (long long)y * byte_count_line, - frame.image_data(0).data() + (long long)y * byte_count_line, - byte_count_line); + + if (hdr) { + const int NUM_THREADS = 4; + auto rows_per_thread = decklink_format_desc.height / NUM_THREADS; + size_t byte_count_line = get_row_bytes(decklink_format_desc, hdr); + tbb::parallel_for(0, NUM_THREADS, [&](int i) { + auto end = (i + 1) * rows_per_thread; + for (int y = firstLine + i * rows_per_thread; y < end; y += decklink_format_desc.field_count) { + auto dest = reinterpret_cast(image_data.get()) + (long long)y * byte_count_line / 4; + for (int x = 0; x < decklink_format_desc.width; x += 1) { + auto src = reinterpret_cast( + frame.image_data(0).data() + (long long)y * decklink_format_desc.width * 8 + x * 8); + uint16_t blue = src[0] >> 6; + uint16_t green = src[1] >> 6; + uint16_t red = src[2] >> 6; + dest[x] = ((uint32_t)(red) << 22) + ((uint32_t)(green) << 12) + ((uint32_t)(blue) << 2); + } + } + }); + + } else { + size_t byte_count_line = (size_t)decklink_format_desc.width * 4; + for (int y = firstLine; y < decklink_format_desc.height; y += decklink_format_desc.field_count) { + std::memcpy(reinterpret_cast(image_data.get()) + (long long)y * byte_count_line, + frame.image_data(0).data() + (long long)y * byte_count_line, + byte_count_line); + } } } else { // Take a sub-region @@ -131,9 +168,10 @@ std::shared_ptr convert_frame_for_port(const core::video_format_desc& chan const port_configuration& config, const core::const_frame& frame1, const core::const_frame& frame2, - BMDFieldDominance field_dominance) + BMDFieldDominance field_dominance, + bool hdr) { - std::shared_ptr image_data = create_aligned_buffer(decklink_format_desc.size); + std::shared_ptr image_data = allocate_frame_data(decklink_format_desc, hdr); if (field_dominance != bmdProgressiveFrame) { convert_frame(channel_format_desc, @@ -141,17 +179,19 @@ std::shared_ptr convert_frame_for_port(const core::video_format_desc& chan config, image_data, field_dominance == bmdUpperFieldFirst, - frame1); + frame1, + hdr); convert_frame(channel_format_desc, decklink_format_desc, config, image_data, field_dominance != bmdUpperFieldFirst, - frame2); + frame2, + hdr); } else { - convert_frame(channel_format_desc, decklink_format_desc, config, image_data, true, frame1); + convert_frame(channel_format_desc, decklink_format_desc, config, image_data, true, frame1, hdr); } if (config.key_only) { @@ -161,4 +201,4 @@ std::shared_ptr convert_frame_for_port(const core::video_format_desc& chan return image_data; } -}} // namespace caspar::decklink \ No newline at end of file +}} // namespace caspar::decklink diff --git a/src/modules/decklink/consumer/frame.h b/src/modules/decklink/consumer/frame.h index 199c4ec4c3..e96109c4ca 100644 --- a/src/modules/decklink/consumer/frame.h +++ b/src/modules/decklink/consumer/frame.h @@ -34,11 +34,17 @@ namespace caspar { namespace decklink { +BMDPixelFormat get_pixel_format(bool hdr); +int get_row_bytes(const core::video_format_desc& format_desc, bool hdr); + +std::shared_ptr allocate_frame_data(const core::video_format_desc& format_desc, bool hdr); + std::shared_ptr convert_frame_for_port(const core::video_format_desc& channel_format_desc, const core::video_format_desc& decklink_format_desc, const port_configuration& config, const core::const_frame& frame1, const core::const_frame& frame2, - BMDFieldDominance field_dominance); + BMDFieldDominance field_dominance, + bool hdr); -}} // namespace caspar::decklink \ No newline at end of file +}} // namespace caspar::decklink From b2494934a361d32e1e86a1534f13992005cad898 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Tue, 23 Apr 2024 08:27:12 +0200 Subject: [PATCH 07/21] Add support for higher color depth in decklink producer --- .../decklink/producer/decklink_producer.cpp | 163 ++++++++++++++---- 1 file changed, 127 insertions(+), 36 deletions(-) diff --git a/src/modules/decklink/producer/decklink_producer.cpp b/src/modules/decklink/producer/decklink_producer.cpp index 53b1825e43..0615e5d354 100644 --- a/src/modules/decklink/producer/decklink_producer.cpp +++ b/src/modules/decklink/producer/decklink_producer.cpp @@ -87,7 +87,8 @@ struct Filter Filter(std::string filter_spec, AVMediaType type, const core::video_format_desc& format_desc, - const com_ptr& dm) + const com_ptr& dm, + bool hdr) { BMDTimeScale timeScale; BMDTimeValue frameDuration; @@ -174,6 +175,7 @@ struct Filter FF(avfilter_graph_parse2(graph.get(), filter_spec.c_str(), &inputs, &outputs)); + auto pix_fmt = (hdr ? AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_UYVY422); for (auto cur = inputs; cur; cur = cur->next) { const auto filter_type = avfilter_pad_get_type(cur->filter_ctx->input_pads, cur->pad_idx); @@ -187,7 +189,7 @@ struct Filter auto args = (boost::format("video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:sar=%d/%d:frame_rate=%d/%d") % - dm->GetWidth() % dm->GetHeight() % AV_PIX_FMT_UYVY422 % 1 % AV_TIME_BASE % sar.numerator() % + dm->GetWidth() % dm->GetHeight() % pix_fmt % 1 % AV_TIME_BASE % sar.numerator() % sar.denominator() % (timeScale / 1000 * (dm->GetFieldDominance() == bmdProgressiveFrame ? 1 : 2)) % (frameDuration / 1000)) .str(); @@ -225,7 +227,7 @@ struct Filter #pragma warning(push) #pragma warning(disable : 4245) #endif - AVPixelFormat pix_fmts[] = {AV_PIX_FMT_UYVY422, AV_PIX_FMT_NONE}; + AVPixelFormat pix_fmts[] = {pix_fmt, AV_PIX_FMT_NONE }; FF(av_opt_set_int_list(sink, "pix_fmts", pix_fmts, -1, AV_OPT_SEARCH_CHILDREN)); #ifdef _MSC_VER #pragma warning(pop) @@ -273,6 +275,88 @@ struct Filter } }; +struct Decoder +{ + Decoder(const Decoder&) = delete; + + bool hdr_ = false; + + public: + std::shared_ptr ctx; + + Decoder() = default; + + explicit Decoder(bool hdr, const com_ptr& mode) + : hdr_(hdr) + { + const auto codec = avcodec_find_decoder(AV_CODEC_ID_V210); + if (!codec) { + FF_RET(AVERROR_DECODER_NOT_FOUND, "avcodec_find_decoder"); + } + + ctx = std::shared_ptr(avcodec_alloc_context3(codec), + [](AVCodecContext* ptr) { avcodec_free_context(&ptr); }); + if (!ctx) { + FF_RET(AVERROR(ENOMEM), "avcodec_alloc_context3"); + } + + auto params = std::shared_ptr(avcodec_parameters_alloc(), + [](AVCodecParameters* ptr) { avcodec_parameters_free(&ptr); }); + if (!params) { + FF_RET(AVERROR(ENOMEM), "avcodec_parameters_alloc"); + } + params->width = mode->GetWidth(); + params->height = mode->GetHeight(); + params->codec_type = AVMEDIA_TYPE_VIDEO; + params->codec_id = AV_CODEC_ID_V210; + params->format = AV_PIX_FMT_YUV422P10; + + FF(avcodec_parameters_to_context(ctx.get(), params.get())); + + // int thread_count = env::properties().get(L"configuration.ffmpeg.producer.threads", 0); + FF(av_opt_set_image_size(ctx.get(), "video_size", mode->GetWidth(), mode->GetHeight(), 0)); + FF(avcodec_open2(ctx.get(), codec, nullptr)); + } + + std::shared_ptr decode(IDeckLinkVideoInputFrame* video, const com_ptr& mode) + { + void* video_bytes = nullptr; + if (SUCCEEDED(video->GetBytes(&video_bytes)) && video_bytes) { + video->AddRef(); + + auto frame = std::shared_ptr(av_frame_alloc(), [video](AVFrame* ptr) { + video->Release(); + av_frame_free(&ptr); + }); + if (!frame) + FF_RET(AVERROR(ENOMEM), "av_frame_alloc"); + + if (hdr_) { + const auto size = video->GetRowBytes() * video->GetHeight(); + AVPacket packet; + av_init_packet(&packet); + packet.data = reinterpret_cast(video_bytes); + packet.size = size; + FF(avcodec_send_packet(ctx.get(), &packet)); + FF(avcodec_receive_frame(ctx.get(), frame.get())); + } else { + frame->format = AV_PIX_FMT_UYVY422; + frame->width = video->GetWidth(); + frame->height = video->GetHeight(); + frame->data[0] = reinterpret_cast(video_bytes); + frame->linesize[0] = video->GetRowBytes(); + frame->key_frame = 1; + } + + frame->interlaced_frame = mode->GetFieldDominance() != bmdProgressiveFrame; + frame->top_field_first = mode->GetFieldDominance() == bmdUpperFieldFirst ? 1 : 0; + + return frame; + } + return nullptr; + } +}; + com_ptr get_display_mode(const com_iface_ptr& device, BMDDisplayMode format, BMDPixelFormat pix_fmt, @@ -316,6 +400,8 @@ static com_ptr get_display_mode(const com_iface_ptrEnableVideoInput(mode_->GetDisplayMode(), bmdFormat8BitYUV, flags))) { + if (FAILED(input_->EnableVideoInput(mode_->GetDisplayMode(), get_pixel_format2(hdr_), flags))) { CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable video input.") << boost::errinfo_api_function("EnableVideoInput")); } @@ -465,15 +557,16 @@ class decklink_producer : public IDeckLinkInputCallback // reinitializing filters because not all filters can handle on-the-fly format changes input_format = new_fmt; - mode_ = get_display_mode(input_, newMode, bmdFormat8BitYUV, bmdSupportedVideoModeDefault); + mode_ = get_display_mode(input_, newMode, get_pixel_format2(hdr_), bmdSupportedVideoModeDefault); graph_->set_text(print()); - video_filter_ = Filter(vfilter_, AVMEDIA_TYPE_VIDEO, format_desc_, mode_); - audio_filter_ = Filter(afilter_, AVMEDIA_TYPE_AUDIO, format_desc_, mode_); + video_filter_ = Filter(vfilter_, AVMEDIA_TYPE_VIDEO, format_desc_, mode_, hdr_); + audio_filter_ = Filter(afilter_, AVMEDIA_TYPE_AUDIO, format_desc_, mode_, hdr_); // reinitializing video input with the new display mode - if (FAILED(input_->EnableVideoInput(newMode, bmdFormat8BitYUV, bmdVideoInputEnableFormatDetection))) { + if (FAILED( + input_->EnableVideoInput(newMode, get_pixel_format2(hdr_), bmdVideoInputEnableFormatDetection))) { CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Unable to enable video input.") << boost::errinfo_api_function("EnableVideoInput")); } @@ -552,27 +645,14 @@ class decklink_producer : public IDeckLinkInputCallback return S_OK; } - auto src = std::shared_ptr(av_frame_alloc(), [](AVFrame* ptr) { av_frame_free(&ptr); }); - src->format = AV_PIX_FMT_UYVY422; - src->width = video->GetWidth(); - src->height = video->GetHeight(); - src->interlaced_frame = mode_->GetFieldDominance() != bmdProgressiveFrame; - src->top_field_first = mode_->GetFieldDominance() == bmdUpperFieldFirst ? 1 : 0; - src->key_frame = 1; - - void* video_bytes = nullptr; - if (SUCCEEDED(video->GetBytes(&video_bytes)) && video_bytes) { - video->AddRef(); - src = std::shared_ptr(src.get(), [src, video](AVFrame* ptr) { video->Release(); }); - - src->data[0] = reinterpret_cast(video_bytes); - src->linesize[0] = video->GetRowBytes(); - - BMDTimeValue duration; - if (SUCCEEDED(video->GetStreamTime(&in_video_pts, &duration, AV_TIME_BASE))) { - src->pts = in_video_pts; - } + auto src = video_decoder_.decode(video, mode_); + + BMDTimeValue duration; + if (SUCCEEDED(video->GetStreamTime(&in_video_pts, &duration, AV_TIME_BASE))) { + src->pts = in_video_pts; + } + if (src) { if (video_filter_.video_source) { FF(av_buffersrc_write_frame(video_filter_.video_source, src.get())); } @@ -761,7 +841,8 @@ class decklink_producer_proxy : public core::frame_producer const std::string& afilter, uint32_t length, const std::wstring& format, - bool freeze_on_lost) + bool freeze_on_lost, + bool hdr) : length_(length) , executor_(L"decklink_producer[" + std::to_wstring(device_index) + L"]") { @@ -769,8 +850,15 @@ class decklink_producer_proxy : public core::frame_producer executor_.invoke([=] { core::diagnostics::call_context::for_thread() = ctx; com_initialize(); - producer_.reset(new decklink_producer( - format_desc, device_index, frame_factory, format_repository, vfilter, afilter, format, freeze_on_lost)); + producer_.reset(new decklink_producer(format_desc, + device_index, + frame_factory, + format_repository, + vfilter, + afilter, + format, + freeze_on_lost, + hdr)); }); } @@ -819,6 +907,8 @@ spl::shared_ptr create_producer(const core::frame_producer auto freeze_on_lost = contains_param(L"FREEZE_ON_LOST", params); + auto hdr = contains_param(L"HDR", params); + auto format_str = get_param(L"FORMAT", params); auto filter_str = get_param(L"FILTER", params); @@ -839,7 +929,8 @@ spl::shared_ptr create_producer(const core::frame_producer u8(afilter), length, format_str, - freeze_on_lost); + freeze_on_lost, + hdr); return core::create_destroy_proxy(producer); } }} // namespace caspar::decklink From fe74379e2a3efb27c61140da480367a019bf777b Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:43:06 +0200 Subject: [PATCH 08/21] Add uniforms for color matrix and luma coefficients * Remove the 'magic numbers' related to YCbCr->RGB conversion for SD / HD * Replace with standard coefficients and matrices for bt.601 and bt.709 respectively --- src/accelerator/ogl/image/image_kernel.cpp | 12 ++++++- src/accelerator/ogl/image/shader.frag | 41 ++++++---------------- src/accelerator/ogl/util/shader.cpp | 12 +++++++ src/accelerator/ogl/util/shader.h | 2 ++ 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/accelerator/ogl/image/image_kernel.cpp b/src/accelerator/ogl/image/image_kernel.cpp index 40d0ede789..bc9651e78a 100644 --- a/src/accelerator/ogl/image/image_kernel.cpp +++ b/src/accelerator/ogl/image/image_kernel.cpp @@ -254,6 +254,11 @@ struct image_kernel::impl params.layer_key->bind(static_cast(texture_id::layer_key)); } + const auto is_hd = params.pix_desc.planes.at(0).height > 700; + + const float color_matrix_sd[9] = {1.0, 0.0, 1.402, 1.0, -0.344, -0.509, 1.0, 1.772, 0.0}; + const float color_matrix_hd[9] = {1.0, 0.0, 1.5748, 1.0, -0.1873, -0.4681, 1.0, 1.8556, 0.0}; + // Setup shader shader_->use(); @@ -268,7 +273,12 @@ struct image_kernel::impl shader_->set("precision_factor[3]", precision_factor[3]); shader_->set("local_key", texture_id::local_key); shader_->set("layer_key", texture_id::layer_key); - shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0); + shader_->set_matrix3("color_matrix", is_hd ? color_matrix_hd : color_matrix_sd); + if (is_hd) { + shader_->set("luma_coeff", 0.2126, 0.7152, 0.0722); + } else { + shader_->set("luma_coeff", 0.299, 0.587, 0.114); + } shader_->set("has_local_key", static_cast(params.local_key)); shader_->set("has_layer_key", static_cast(params.layer_key)); shader_->set("pixel_format", params.pix_desc.format); diff --git a/src/accelerator/ogl/image/shader.frag b/src/accelerator/ogl/image/shader.frag index e277d792f7..af04f283af 100644 --- a/src/accelerator/ogl/image/shader.frag +++ b/src/accelerator/ogl/image/shader.frag @@ -8,7 +8,8 @@ uniform sampler2D plane[4]; uniform sampler2D local_key; uniform sampler2D layer_key; -uniform bool is_hd; +uniform mat3 color_matrix; +uniform vec3 luma_coeff; uniform bool has_local_key; uniform bool has_layer_key; uniform int blend_mode; @@ -52,9 +53,7 @@ vec3 ContrastSaturationBrightness(vec4 color, float brt, float sat, float con) const float AvgLumG = 0.5; const float AvgLumB = 0.5; - vec3 LumCoeff = is_hd - ? vec3(0.0722, 0.7152, 0.2126) - : vec3(0.114, 0.587, 0.299); + vec3 LumCoeff = luma_coeff.bgr; if (color.a > 0.0) color.rgb /= color.a; @@ -290,9 +289,7 @@ vec3 BlendLuminosity(vec3 base, vec3 blend) // by F. van den Bergh & V. Lalioti // but as a pixel shader algorithm. // -vec4 grey_xfer = is_hd - ? vec4(0.2126, 0.7152, 0.0722, 0) - : vec4(0.299, 0.587, 0.114, 0); +vec4 grey_xfer = vec4(luma_coeff, 0); // This allows us to implement the paper's alphaMap curve in software // rather than a largeish array @@ -439,32 +436,16 @@ vec4 chroma_key(vec4 c) return ChromaOnCustomColor(c.bgra).bgra; } -vec4 ycbcra_to_rgba_sd(float Y, float Cb, float Cr, float A) +vec4 ycbcra_to_rgba(float Y, float Cb, float Cr, float A) { - vec4 rgba; - rgba.b = (1.164*(Y*255 - 16) + 1.596*(Cr*255 - 128))/255; - rgba.g = (1.164*(Y*255 - 16) - 0.813*(Cr*255 - 128) - 0.391*(Cb*255 - 128))/255; - rgba.r = (1.164*(Y*255 - 16) + 2.018*(Cb*255 - 128))/255; - rgba.a = A; - return rgba; -} + const float luma_coefficient = 255.0/219.0; + const float chroma_coefficient = 255.0/224.0; -vec4 ycbcra_to_rgba_hd(float Y, float Cb, float Cr, float A) -{ - vec4 rgba; - rgba.b = (1.164*(Y*255 - 16) + 1.793*(Cr*255 - 128))/255; - rgba.g = (1.164*(Y*255 - 16) - 0.534*(Cr*255 - 128) - 0.213*(Cb*255 - 128))/255; - rgba.r = (1.164*(Y*255 - 16) + 2.115*(Cb*255 - 128))/255; - rgba.a = A; - return rgba; -} + vec3 YCbCr = vec3(Y, Cb, Cr) * 255; + YCbCr -= vec3(16.0, 128.0, 128.0); + YCbCr *= vec3(luma_coefficient, chroma_coefficient, chroma_coefficient); -vec4 ycbcra_to_rgba(float y, float cb, float cr, float a) -{ - if(is_hd) - return ycbcra_to_rgba_hd(y, cb, cr, a); - else - return ycbcra_to_rgba_sd(y, cb, cr, a); + return vec4(color_matrix * YCbCr / 255, A).bgra; } vec4 get_sample(sampler2D sampler, vec2 coords) diff --git a/src/accelerator/ogl/util/shader.cpp b/src/accelerator/ogl/util/shader.cpp index 8e9acc0611..308903790d 100644 --- a/src/accelerator/ogl/util/shader.cpp +++ b/src/accelerator/ogl/util/shader.cpp @@ -127,11 +127,21 @@ struct shader::impl { GL(glUniform2f(get_uniform_location(name.c_str()), static_cast(value0), static_cast(value1))); } + void set(const std::string& name, double value0, double value1, double value2) { + GL(glUniform3f(get_uniform_location(name.c_str()), + static_cast(value0), + static_cast(value1), + static_cast(value1))); + } void set(const std::string& name, double value) { GL(glUniform1f(get_uniform_location(name.c_str()), static_cast(value))); } + void set_matrix3(const std::string& name, const float* value) + { + GL(glUniformMatrix3fv(get_uniform_location(name.c_str()), 1, GL_TRUE, value)); + } void use() { GL(glUseProgramObjectARB(program_)); } }; @@ -145,7 +155,9 @@ void shader::set(const std::string& name, bool value) { impl_->set(name, value) void shader::set(const std::string& name, int value) { impl_->set(name, value); } void shader::set(const std::string& name, float value) { impl_->set(name, value); } void shader::set(const std::string& name, double value0, double value1) { impl_->set(name, value0, value1); } +void shader::set(const std::string& name, double value0, double value1, double value2) { impl_->set(name, value0, value1, value2); } void shader::set(const std::string& name, double value) { impl_->set(name, value); } +void shader::set_matrix3(const std::string& name, const float* value) { impl_->set_matrix3(name, value); } GLint shader::get_attrib_location(const char* name) { return impl_->get_attrib_location(name); } int shader::id() const { return impl_->program_; } void shader::use() const { impl_->use(); } diff --git a/src/accelerator/ogl/util/shader.h b/src/accelerator/ogl/util/shader.h index 67fe885a88..6d80a54bf1 100644 --- a/src/accelerator/ogl/util/shader.h +++ b/src/accelerator/ogl/util/shader.h @@ -41,7 +41,9 @@ class shader final void set(const std::string& name, int value); void set(const std::string& name, float value); void set(const std::string& name, double value0, double value1); + void set(const std::string& name, double value0, double value1, double value2); void set(const std::string& name, double value); + void set_matrix3(const std::string& name, const float* value); GLint get_attrib_location(const char* name); From a1c5e5f4e583ed3d57c0b9e48ca40238e78e985c Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:45:48 +0200 Subject: [PATCH 09/21] Add color_space property to pixel_format_desc --- src/core/frame/pixel_format.h | 11 ++++++++++- src/modules/ffmpeg/util/av_util.cpp | 14 ++++++++++---- src/modules/ffmpeg/util/av_util.h | 9 +++++++-- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/core/frame/pixel_format.h b/src/core/frame/pixel_format.h index 338e9d85fc..520e24c0bd 100644 --- a/src/core/frame/pixel_format.h +++ b/src/core/frame/pixel_format.h @@ -44,6 +44,13 @@ enum class pixel_format invalid, }; +enum class color_space +{ + bt601, + bt709, + bt2020, +}; + struct pixel_format_desc final { struct plane @@ -70,13 +77,15 @@ struct pixel_format_desc final pixel_format_desc() = default; - explicit pixel_format_desc(pixel_format format) + explicit pixel_format_desc(pixel_format format, core::color_space color_space = core::color_space::bt709) : format(format) + , color_space(color_space) { } pixel_format format = pixel_format::invalid; std::vector planes; + core::color_space color_space = core::color_space::bt709; }; }} // namespace caspar::core diff --git a/src/modules/ffmpeg/util/av_util.cpp b/src/modules/ffmpeg/util/av_util.cpp index b853f5b7db..4eadbb1769 100644 --- a/src/modules/ffmpeg/util/av_util.cpp +++ b/src/modules/ffmpeg/util/av_util.cpp @@ -46,12 +46,14 @@ std::shared_ptr alloc_packet() core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory, std::shared_ptr video, - std::shared_ptr audio) + std::shared_ptr audio, + core::color_space color_space) { std::vector data_map; // TODO(perf) when using data_map, avoid uploading duplicate planes const auto pix_desc = - video ? pixel_format_desc(static_cast(video->format), video->width, video->height, data_map) + video ? pixel_format_desc( + static_cast(video->format), video->width, video->height, data_map, color_space) : core::pixel_format_desc(core::pixel_format::invalid); auto frame = frame_factory.create_frame(tag, pix_desc); @@ -144,14 +146,18 @@ std::tuple get_pixel_format(AVPixelFormat } } -core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int height, std::vector& data_map) +core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, + int width, + int height, + std::vector& data_map, + core::color_space color_space) { // Get linesizes int linesizes[4]; av_image_fill_linesizes(linesizes, pix_fmt, width); const auto fmt = get_pixel_format(pix_fmt); - auto desc = core::pixel_format_desc(std::get<0>(fmt)); + auto desc = core::pixel_format_desc(std::get<0>(fmt), color_space); auto depth = std::get<1>(fmt); switch (desc.format) { diff --git a/src/modules/ffmpeg/util/av_util.h b/src/modules/ffmpeg/util/av_util.h index c7e5f2ff62..fd92d99192 100644 --- a/src/modules/ffmpeg/util/av_util.h +++ b/src/modules/ffmpeg/util/av_util.h @@ -20,11 +20,16 @@ namespace caspar { namespace ffmpeg { std::shared_ptr alloc_frame(); std::shared_ptr alloc_packet(); -core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int height, std::vector& data_map); +core::pixel_format_desc + pixel_format_desc(AVPixelFormat pix_fmt, + int width, + int height, + std::vector& data_map, core::color_space color_space = core::color_space::bt709); core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory, std::shared_ptr video, - std::shared_ptr audio); + std::shared_ptr audio, + core::color_space color_space = core::color_space::bt709); std::shared_ptr make_av_video_frame(const core::const_frame& frame, const core::video_format_desc& format_des); std::shared_ptr make_av_audio_frame(const core::const_frame& frame, const core::video_format_desc& format_des); From 1140077bb5027ad8be40aa1d69d899a40dc3c46a Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:16:46 +0200 Subject: [PATCH 10/21] Add support for bt.2020 in image kernel --- src/accelerator/ogl/image/image_kernel.cpp | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/accelerator/ogl/image/image_kernel.cpp b/src/accelerator/ogl/image/image_kernel.cpp index bc9651e78a..d8544e053d 100644 --- a/src/accelerator/ogl/image/image_kernel.cpp +++ b/src/accelerator/ogl/image/image_kernel.cpp @@ -253,14 +253,21 @@ struct image_kernel::impl if (params.layer_key) { params.layer_key->bind(static_cast(texture_id::layer_key)); } - + const auto is_hd = params.pix_desc.planes.at(0).height > 700; + const auto color_space = is_hd ? params.pix_desc.color_space : core::color_space::bt601; - const float color_matrix_sd[9] = {1.0, 0.0, 1.402, 1.0, -0.344, -0.509, 1.0, 1.772, 0.0}; - const float color_matrix_hd[9] = {1.0, 0.0, 1.5748, 1.0, -0.1873, -0.4681, 1.0, 1.8556, 0.0}; + const float color_matrices[3][9] = {{1.0, 0.0, 1.402, 1.0, -0.344, -0.509, 1.0, 1.772, 0.0}, //bt.601 + {1.0, 0.0, 1.5748, 1.0, -0.1873, -0.4681, 1.0, 1.8556, 0.0}, //bt.709 + {1.0, 0.0, 1.4746, 1.0, -0.16455312684366, -0.57135312684366, 1.0, 1.8814, 0.0}}; //bt.2020 + const auto color_matrix = color_matrices[static_cast(color_space)]; - // Setup shader + const float luma_coefficients[3][3] = {{0.299, 0.587, 0.114}, //bt.601 + {0.2126, 0.7152, 0.0722}, //bt.709 + {0.2627, 0.6780, 0.0593}}; //bt.2020 + const auto luma_coeff = luma_coefficients[static_cast(color_space)]; + // Setup shader shader_->use(); shader_->set("plane[0]", texture_id::plane0); @@ -273,12 +280,8 @@ struct image_kernel::impl shader_->set("precision_factor[3]", precision_factor[3]); shader_->set("local_key", texture_id::local_key); shader_->set("layer_key", texture_id::layer_key); - shader_->set_matrix3("color_matrix", is_hd ? color_matrix_hd : color_matrix_sd); - if (is_hd) { - shader_->set("luma_coeff", 0.2126, 0.7152, 0.0722); - } else { - shader_->set("luma_coeff", 0.299, 0.587, 0.114); - } + shader_->set_matrix3("color_matrix", color_matrix); + shader_->set("luma_coeff", luma_coeff[0], luma_coeff[1], luma_coeff[2]); shader_->set("has_local_key", static_cast(params.local_key)); shader_->set("has_layer_key", static_cast(params.layer_key)); shader_->set("pixel_format", params.pix_desc.format); From de98ca023d7a4126fc6c56a169f7219001e20c95 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:39:45 +0200 Subject: [PATCH 11/21] Add color_space property to image_mixer * Needed for providing necessary metadata to consumers about the frame's color_space * color_space is deduced from the color-depth channel config. A separate config could be added later * The fragment shader does not perform any color space conversions --- src/accelerator/accelerator.cpp | 9 ++++---- src/accelerator/accelerator.h | 4 +++- src/accelerator/ogl/image/image_mixer.cpp | 25 ++++++++++++++++++----- src/accelerator/ogl/image/image_mixer.h | 4 +++- src/core/mixer/image/image_mixer.h | 2 ++ src/core/mixer/mixer.cpp | 4 +++- src/shell/server.cpp | 4 +++- 7 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/accelerator/accelerator.cpp b/src/accelerator/accelerator.cpp index 98196c6d3b..02e9d21f10 100644 --- a/src/accelerator/accelerator.cpp +++ b/src/accelerator/accelerator.cpp @@ -25,10 +25,11 @@ struct accelerator::impl { } - std::unique_ptr create_image_mixer(int channel_id, common::bit_depth depth) + std::unique_ptr + create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space) { return std::make_unique( - spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size(), depth); + spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size(), depth, color_space); } std::shared_ptr get_device() @@ -48,9 +49,9 @@ accelerator::accelerator(const core::video_format_repository format_repository) accelerator::~accelerator() {} -std::unique_ptr accelerator::create_image_mixer(const int channel_id, common::bit_depth depth) +std::unique_ptr accelerator::create_image_mixer(const int channel_id, common::bit_depth depth, core::color_space color_space) { - return impl_->create_image_mixer(channel_id, depth); + return impl_->create_image_mixer(channel_id, depth, color_space); } std::shared_ptr accelerator::get_device() const diff --git a/src/accelerator/accelerator.h b/src/accelerator/accelerator.h index f7419d7f99..abf5ba6b3f 100644 --- a/src/accelerator/accelerator.h +++ b/src/accelerator/accelerator.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -29,7 +30,8 @@ class accelerator accelerator& operator=(accelerator&) = delete; - std::unique_ptr create_image_mixer(int channel_id, common::bit_depth depth); + std::unique_ptr + create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space); std::shared_ptr get_device() const; diff --git a/src/accelerator/ogl/image/image_mixer.cpp b/src/accelerator/ogl/image/image_mixer.cpp index e34f299503..287f56e27a 100644 --- a/src/accelerator/ogl/image/image_mixer.cpp +++ b/src/accelerator/ogl/image/image_mixer.cpp @@ -72,13 +72,18 @@ class image_renderer image_kernel kernel_; const size_t max_frame_size_; common::bit_depth depth_; + core::color_space color_space_; public: - explicit image_renderer(const spl::shared_ptr& ogl, const size_t max_frame_size, common::bit_depth depth) + explicit image_renderer(const spl::shared_ptr& ogl, + const size_t max_frame_size, + common::bit_depth depth, + core::color_space color_space) : ogl_(ogl) , kernel_(ogl_) , max_frame_size_(max_frame_size) , depth_(depth) + , color_space_(color_space) { } @@ -100,6 +105,7 @@ class image_renderer } common::bit_depth depth() const { return depth_; } + core::color_space color_space() const { return color_space_; } private: void draw(std::shared_ptr& target_texture, @@ -162,6 +168,8 @@ class image_renderer const core::video_format_desc& format_desc) { draw_params draw_params; + // TODO: Pass the target color_space + draw_params.pix_desc = std::move(item.pix_desc); draw_params.transform = std::move(item.transform); draw_params.geometry = item.geometry; @@ -237,9 +245,13 @@ struct image_mixer::impl std::vector layer_stack_; public: - impl(const spl::shared_ptr& ogl, const int channel_id, const size_t max_frame_size, common::bit_depth depth) + impl(const spl::shared_ptr& ogl, + const int channel_id, + const size_t max_frame_size, + common::bit_depth depth, + core::color_space color_space) : ogl_(ogl) - , renderer_(ogl, max_frame_size, depth) + , renderer_(ogl, max_frame_size, depth, color_space) , transform_stack_(1) { CASPAR_LOG(info) << L"Initialized OpenGL Accelerated GPU Image Mixer for channel " << channel_id; @@ -344,13 +356,15 @@ struct image_mixer::impl } common::bit_depth depth() const { return renderer_.depth(); } + core::color_space color_space() const { return renderer_.color_space(); } }; image_mixer::image_mixer(const spl::shared_ptr& ogl, const int channel_id, const size_t max_frame_size, - common::bit_depth depth) - : impl_(std::make_unique(ogl, channel_id, max_frame_size, depth)) + common::bit_depth depth, + core::color_space color_space) + : impl_(std::make_unique(ogl, channel_id, max_frame_size, depth, color_space)) { } image_mixer::~image_mixer() {} @@ -372,5 +386,6 @@ image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, } common::bit_depth image_mixer::depth() const { return impl_->depth(); } +core::color_space image_mixer::color_space() const { return impl_->color_space(); } }}} // namespace caspar::accelerator::ogl diff --git a/src/accelerator/ogl/image/image_mixer.h b/src/accelerator/ogl/image/image_mixer.h index 609a7e8d11..0d90dc6fa5 100644 --- a/src/accelerator/ogl/image/image_mixer.h +++ b/src/accelerator/ogl/image/image_mixer.h @@ -40,7 +40,8 @@ class image_mixer final : public core::image_mixer image_mixer(const spl::shared_ptr& ogl, int channel_id, const size_t max_frame_size, - common::bit_depth depth); + common::bit_depth depth, + core::color_space color_space); image_mixer(const image_mixer&) = delete; ~image_mixer(); @@ -58,6 +59,7 @@ class image_mixer final : public core::image_mixer void visit(const core::const_frame& frame) override; void pop() override; common::bit_depth depth() const override; + core::color_space color_space() const override; private: struct impl; diff --git a/src/core/mixer/image/image_mixer.h b/src/core/mixer/image/image_mixer.h index 795bec1bcb..7906391939 100644 --- a/src/core/mixer/image/image_mixer.h +++ b/src/core/mixer/image/image_mixer.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -53,6 +54,7 @@ class image_mixer common::bit_depth depth) override = 0; virtual common::bit_depth depth() const = 0; + virtual core::color_space color_space() const = 0; }; }} // namespace caspar::core diff --git a/src/core/mixer/mixer.cpp b/src/core/mixer/mixer.cpp index fbf606b90d..847f9b31cf 100644 --- a/src/core/mixer/mixer.cpp +++ b/src/core/mixer/mixer.cpp @@ -74,15 +74,17 @@ struct mixer::impl state_["audio"] = audio_mixer_.state(); auto depth = image_mixer_->depth(); + auto color_space = image_mixer_->color_space(); buffer_.push(std::async(std::launch::deferred, [image = std::move(image), audio = std::move(audio), graph = graph_, depth, + color_space, format_desc, tag = this]() mutable { - auto desc = pixel_format_desc(pixel_format::bgra); + auto desc = pixel_format_desc(pixel_format::bgra, color_space); desc.planes.push_back( pixel_format_desc::plane(format_desc.width, format_desc.height, 4, depth)); std::vector> image_data; diff --git a/src/shell/server.cpp b/src/shell/server.cpp index 2116891f34..cc87248840 100644 --- a/src/shell/server.cpp +++ b/src/shell/server.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -267,10 +268,11 @@ struct server::impl auto weak_client = std::weak_ptr(osc_client_); auto channel_id = static_cast(channels_->size() + 1); auto depth = color_depth == 16 ? common::bit_depth::bit16 : common::bit_depth::bit8; + auto color_space = depth == common::bit_depth::bit16 ? core::color_space::bt2020 : core::color_space::bt709; auto channel = spl::make_shared(channel_id, format_desc, - accelerator_.create_image_mixer(channel_id, depth), + accelerator_.create_image_mixer(channel_id, depth, color_space), [channel_id, weak_client](core::monitor::state channel_state) { monitor::state state; state[""]["channel"][channel_id] = channel_state; From 0db73ded8330c85efed0a1e9fc40a5d7bac4bf68 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:42:08 +0200 Subject: [PATCH 12/21] Add correct color space metadata to frames from the decklink producer --- src/modules/decklink/decklink_api.h | 1 + .../decklink/producer/decklink_producer.cpp | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/modules/decklink/decklink_api.h b/src/modules/decklink/decklink_api.h index c0553a46df..1749976c61 100644 --- a/src/modules/decklink/decklink_api.h +++ b/src/modules/decklink/decklink_api.h @@ -119,6 +119,7 @@ using BOOL = bool; #define TRUE true #define FALSE false using UINT32 = uint32_t; +using LONGLONG = int64_t; static std::wstring to_string(String utf16_string) { return u16(utf16_string); } diff --git a/src/modules/decklink/producer/decklink_producer.cpp b/src/modules/decklink/producer/decklink_producer.cpp index 0615e5d354..157ee85b5c 100644 --- a/src/modules/decklink/producer/decklink_producer.cpp +++ b/src/modules/decklink/producer/decklink_producer.cpp @@ -357,6 +357,23 @@ struct Decoder } }; +core::color_space get_color_space(IDeckLinkVideoInputFrame* video) +{ + IDeckLinkVideoFrameMetadataExtensions* md = nullptr; + + if (SUCCEEDED(video->QueryInterface(IID_IDeckLinkVideoFrameMetadataExtensions, (void**)&md))) { + auto metadata = wrap_raw(md, true); + LONGLONG color_space; + if (SUCCEEDED(md->GetInt(bmdDeckLinkFrameMetadataColorspace, &color_space))) { + if (color_space == bmdColorspaceRec2020) { + return core::color_space::bt2020; + } + } + } + + return core::color_space::bt709; +} + com_ptr get_display_mode(const com_iface_ptr& device, BMDDisplayMode format, BMDPixelFormat pix_fmt, @@ -628,6 +645,7 @@ class decklink_producer : public IDeckLinkInputCallback BMDTimeValue in_video_pts = 0LL; BMDTimeValue in_audio_pts = 0LL; + core::color_space color_space = core::color_space::bt709; // If the video is delayed too much, audio only will be delivered // we don't want audio only since the buffer is small and we keep avcodec @@ -645,6 +663,7 @@ class decklink_producer : public IDeckLinkInputCallback return S_OK; } + color_space = get_color_space(video); auto src = video_decoder_.decode(video, mode_); BMDTimeValue duration; @@ -739,7 +758,7 @@ class decklink_producer : public IDeckLinkInputCallback graph_->set_value("in-sync", in_sync * 2.0 + 0.5); graph_->set_value("out-sync", out_sync * 2.0 + 0.5); - auto frame = core::draw_frame(make_frame(this, *frame_factory_, av_video, av_audio)); + auto frame = core::draw_frame(make_frame(this, *frame_factory_, av_video, av_audio, color_space)); auto field = core::video_field::progressive; if (format_desc_.field_count == 2) { field = frame_count_ % 2 == 0 ? core::video_field::a : core::video_field::b; From 8d5b7d7f14bf15b402082da83186ce9acb1df5a1 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 29 Apr 2024 06:19:28 +0200 Subject: [PATCH 13/21] Add correct color space metadata to frames from the ffmpeg producer --- src/modules/ffmpeg/producer/av_producer.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/modules/ffmpeg/producer/av_producer.cpp b/src/modules/ffmpeg/producer/av_producer.cpp index f18595bf9e..ada52a5f6d 100644 --- a/src/modules/ffmpeg/producer/av_producer.cpp +++ b/src/modules/ffmpeg/producer/av_producer.cpp @@ -75,6 +75,13 @@ struct Frame // TODO (fix) Handle ts discontinuities. // TODO (feat) Forward options. +core::color_space get_color_space(const std::shared_ptr& video) { + auto result = core::color_space::bt709; + if (video->colorspace == AVColorSpace::AVCOL_SPC_BT2020_NCL) + result = core::color_space::bt2020; + return result; +} + class Decoder { Decoder(const Decoder&) = delete; @@ -868,7 +875,7 @@ struct AVProducer::Impl frame.duration = av_rescale_q(frame.audio->nb_samples, {1, sr}, TIME_BASE_Q); } - frame.frame = core::draw_frame(make_frame(this, *frame_factory_, frame.video, frame.audio)); + frame.frame = core::draw_frame(make_frame(this, *frame_factory_, frame.video, frame.audio, get_color_space(frame.video))); frame.frame_count = frame_count_++; graph_->set_value("decode-time", decode_timer.elapsed() * format_desc_.fps * 0.5); From ebcc6ab22dbfde18ce31d3d595883e909ee71e77 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:02:59 +0200 Subject: [PATCH 14/21] Update decklink consumer HDR metadata --- src/modules/decklink/consumer/config.cpp | 11 +++- src/modules/decklink/consumer/config.h | 10 ++++ .../decklink/consumer/decklink_consumer.cpp | 55 +++++++++---------- src/shell/casparcg.config | 6 ++ 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/src/modules/decklink/consumer/config.cpp b/src/modules/decklink/consumer/config.cpp index 7e2c7ce3bf..71a319e173 100644 --- a/src/modules/decklink/consumer/config.cpp +++ b/src/modules/decklink/consumer/config.cpp @@ -117,6 +117,15 @@ configuration parse_xml_config(const boost::property_tree::wptree& ptree, } } + auto hdr_metadata = ptree.get_child_optional(L"hdr-metadata"); + if(hdr_metadata) + { + config.hdr_meta.min_dml = hdr_metadata->get(L"min-dml", config.hdr_meta.min_dml); + config.hdr_meta.max_dml = hdr_metadata->get(L"max-dml", config.hdr_meta.max_dml); + config.hdr_meta.max_fall = hdr_metadata->get(L"max-fall", config.hdr_meta.max_fall); + config.hdr_meta.max_cll = hdr_metadata->get(L"max-cll", config.hdr_meta.max_cll); + } + return config; } @@ -152,4 +161,4 @@ configuration parse_amcp_config(const std::vector& params, return config; } -}} // namespace caspar::decklink \ No newline at end of file +}} // namespace caspar::decklink diff --git a/src/modules/decklink/consumer/config.h b/src/modules/decklink/consumer/config.h index a5ed339044..d6b97c7599 100644 --- a/src/modules/decklink/consumer/config.h +++ b/src/modules/decklink/consumer/config.h @@ -44,6 +44,14 @@ struct port_configuration } }; +struct hdr_meta_configuration +{ + float min_dml = 0.005f; + float max_dml = 1000.0f; + float max_fall = 100.0f; + float max_cll = 1000.0f; +}; + struct configuration { enum class keyer_t @@ -87,6 +95,8 @@ struct configuration port_configuration primary; std::vector secondaries; + hdr_meta_configuration hdr_meta; + [[nodiscard]] int buffer_depth() const { return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + diff --git a/src/modules/decklink/consumer/decklink_consumer.cpp b/src/modules/decklink/consumer/decklink_consumer.cpp index 00c0c5b265..b0d9200fac 100644 --- a/src/modules/decklink/consumer/decklink_consumer.cpp +++ b/src/modules/decklink/consumer/decklink_consumer.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -213,15 +214,6 @@ struct ChromaticityCoordinates double WhiteY; }; -struct HDRMetadata -{ - int64_t EOTF; - double maxDisplayMasteringLuminance; - double minDisplayMasteringLuminance; - double maxCLL; - double maxFALL; -}; - const auto REC_709 = ChromaticityCoordinates{0.640, 0.330, 0.300, 0.600, 0.150, 0.060, 0.3127, 0.3290}; const auto REC_2020 = ChromaticityCoordinates{0.708, 0.292, 0.170, 0.797, 0.131, 0.046, 0.3127, 0.3290}; @@ -234,15 +226,19 @@ class decklink_frame std::atomic ref_count_{0}; int nb_samples_; const bool hdr_; + core::color_space color_space_; + hdr_meta_configuration hdr_metadata_; BMDFrameFlags flags_; BMDPixelFormat pix_fmt_; public: - decklink_frame(std::shared_ptr data, core::video_format_desc format_desc, int nb_samples, bool hdr) + decklink_frame(std::shared_ptr data, core::video_format_desc format_desc, int nb_samples, bool hdr, core::color_space color_space, const hdr_meta_configuration& hdr_metadata) : format_desc_(std::move(format_desc)) , data_(std::move(data)) , nb_samples_(nb_samples) , hdr_(hdr) + , color_space_(color_space) + , hdr_metadata_(hdr_metadata) , flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault) , pix_fmt_(get_pixel_format(hdr)) { @@ -324,11 +320,11 @@ class decklink_frame switch (metadataID) { case bmdDeckLinkFrameMetadataHDRElectroOpticalTransferFunc: - *value = EOTF::PQ; + *value = EOTF::HLG; break; case bmdDeckLinkFrameMetadataColorspace: - *value = bmdColorspaceRec709; + *value = (color_space_ == core::color_space::bt2020) ? bmdColorspaceRec2020 : bmdColorspaceRec709; break; default: @@ -341,55 +337,56 @@ class decklink_frame HRESULT STDMETHODCALLTYPE GetFloat(BMDDeckLinkFrameMetadataID metadataID, double* value) { + const auto color_space = (color_space_ == core::color_space::bt2020) ? &REC_2020 : &REC_709; HRESULT result = S_OK; switch (metadataID) { case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedX: - *value = REC_709.RedX; + *value = color_space->RedX; break; case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedY: - *value = REC_709.RedY; + *value = color_space->RedY; break; case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenX: - *value = REC_709.GreenX; + *value = color_space->GreenX; break; case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenY: - *value = REC_709.GreenY; + *value = color_space->GreenY; break; case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueX: - *value = REC_709.BlueX; + *value = color_space->BlueX; break; case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueY: - *value = REC_709.BlueY; + *value = color_space->BlueY; break; case bmdDeckLinkFrameMetadataHDRWhitePointX: - *value = REC_709.WhiteX; + *value = color_space->WhiteX; break; case bmdDeckLinkFrameMetadataHDRWhitePointY: - *value = REC_709.WhiteY; + *value = color_space->WhiteY; break; case bmdDeckLinkFrameMetadataHDRMaxDisplayMasteringLuminance: - *value = 1000.0; + *value = hdr_metadata_.max_dml; break; case bmdDeckLinkFrameMetadataHDRMinDisplayMasteringLuminance: - *value = 0.005; + *value = hdr_metadata_.min_dml; break; case bmdDeckLinkFrameMetadataHDRMaximumContentLightLevel: - *value = 1000.0; + *value = hdr_metadata_.max_cll; break; case bmdDeckLinkFrameMetadataHDRMaximumFrameAverageLightLevel: - *value = 50.0; + *value = hdr_metadata_.max_fall; break; default: @@ -558,7 +555,7 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback void schedule_next_video(std::shared_ptr image_data, int nb_samples, BMDTimeValue display_time) { auto packed_frame = wrap_raw( - new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr)); + new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, core::color_space::bt709, config_.hdr_meta)); if (FAILED(output_->ScheduleVideoFrame(get_raw(packed_frame), display_time, decklink_format_desc_.duration, @@ -713,7 +710,7 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback std::shared_ptr image_data = allocate_frame_data(decklink_format_desc_, config_.hdr); - schedule_next_video(image_data, nb_samples, video_scheduled_); + schedule_next_video(image_data, nb_samples, video_scheduled_, core::color_space::bt2020); for (auto& context : secondary_port_contexts_) { context->schedule_next_video(image_data, 0, video_scheduled_); } @@ -935,7 +932,7 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback mode_->GetFieldDominance(), config_.hdr); - schedule_next_video(image_data, nb_samples, video_display_time); + schedule_next_video(image_data, nb_samples, video_display_time, frame1.pixel_format_desc().color_space); if (config_.embedded_audio) { schedule_next_audio(std::move(audio_data), nb_samples); @@ -995,10 +992,10 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback audio_scheduled_ += nb_samples; // TODO - what if there are too many/few samples in this frame? } - void schedule_next_video(std::shared_ptr image_data, int nb_samples, BMDTimeValue display_time) + void schedule_next_video(std::shared_ptr image_data, int nb_samples, BMDTimeValue display_time, core::color_space color_space) { auto fill_frame = wrap_raw( - new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr)); + new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, color_space, config_.hdr_meta)); if (FAILED(output_->ScheduleVideoFrame( get_raw(fill_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) { CASPAR_LOG(error) << print() << L" Failed to schedule primary video."; diff --git a/src/shell/casparcg.config b/src/shell/casparcg.config index f35e79e07a..1ddd1820f9 100644 --- a/src/shell/casparcg.config +++ b/src/shell/casparcg.config @@ -98,6 +98,12 @@ 0 (width of the region to copy. 0 means no-limit) 0 (height of the region to copy. 0 means no-limit) + + 1000 [1..65535] + 1000 [50..65535] + 0.005 [0.0001-6.5535] + 1000 [1-65535] + auto [auto|enable|disable] 10 (seconds) From aac7d44da5e25613efd1a0ffcb53892a45062d8a Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 26 Apr 2024 06:28:15 +0000 Subject: [PATCH 15/21] Add config parameter for channel color-space --- src/shell/casparcg.config | 1 + src/shell/server.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/shell/casparcg.config b/src/shell/casparcg.config index 1ddd1820f9..a961b9d3a1 100644 --- a/src/shell/casparcg.config +++ b/src/shell/casparcg.config @@ -80,6 +80,7 @@ PAL [PAL|NTSC|576p2500|720p2398|720p2400|720p2500|720p5000|720p2997|720p5994|720p3000|720p6000|1080p2398|1080p2400|1080i5000|1080i5994|1080i6000|1080p2500|1080p2997|1080p3000|1080p5000|1080p5994|1080p6000|1556p2398|1556p2400|1556p2500|dci1080p2398|dci1080p2400|dci1080p2500|2160p2398|2160p2400|2160p2500|2160p2997|2160p3000|2160p5000|2160p5994|2160p6000|dci2160p2398|dci2160p2400|dci2160p2500] 8 [8|16] + bt709 [bt709|bt2020] [1..] diff --git a/src/shell/server.cpp b/src/shell/server.cpp index cc87248840..451590e228 100644 --- a/src/shell/server.cpp +++ b/src/shell/server.cpp @@ -261,6 +261,11 @@ struct server::impl if (color_depth != 8 && color_depth != 16) CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Invalid color-depth: " + std::to_wstring(color_depth))); + + auto color_space_str = boost::to_lower_copy(xml_channel.second.get(L"color-space", L"bt709")); + if (color_space_str != L"bt709" && color_space_str != L"bt2020") + CASPAR_THROW_EXCEPTION(user_error() + << msg_info(L"Invalid color-space, must be bt709 or bt2020")); if (format_desc.format == video_format::invalid) CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Invalid video-mode: " + format_desc_str)); @@ -268,7 +273,7 @@ struct server::impl auto weak_client = std::weak_ptr(osc_client_); auto channel_id = static_cast(channels_->size() + 1); auto depth = color_depth == 16 ? common::bit_depth::bit16 : common::bit_depth::bit8; - auto color_space = depth == common::bit_depth::bit16 ? core::color_space::bt2020 : core::color_space::bt709; + auto color_space = color_space_str == L"bt2020" ? core::color_space::bt2020 : core::color_space::bt709; auto channel = spl::make_shared(channel_id, format_desc, From 093bfcb4d46193a841e6ba5d77ab512c1257d8e4 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 09:12:12 +0200 Subject: [PATCH 16/21] [FIXUP] Clean up create_array signature --- src/accelerator/ogl/image/image_mixer.cpp | 5 ++--- src/accelerator/ogl/util/device.cpp | 7 +++---- src/accelerator/ogl/util/device.h | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/accelerator/ogl/image/image_mixer.cpp b/src/accelerator/ogl/image/image_mixer.cpp index 287f56e27a..f99777eacb 100644 --- a/src/accelerator/ogl/image/image_mixer.cpp +++ b/src/accelerator/ogl/image/image_mixer.cpp @@ -327,9 +327,8 @@ struct image_mixer::impl { std::vector> image_data; for (auto& plane : desc.planes) { - image_data.push_back(ogl_->create_array(plane.size, - plane.depth == common::bit_depth::bit8 ? common::bit_depth::bit8 - : common::bit_depth::bit16)); + auto bytes_per_pixel = depth == common::bit_depth::bit8 ? 1 : 2; + image_data.push_back(ogl_->create_array(plane.size * bytes_per_pixel)); } std::weak_ptr weak_self = shared_from_this(); diff --git a/src/accelerator/ogl/util/device.cpp b/src/accelerator/ogl/util/device.cpp index ec3220e0fd..5ceb3c9b0f 100644 --- a/src/accelerator/ogl/util/device.cpp +++ b/src/accelerator/ogl/util/device.cpp @@ -217,10 +217,9 @@ struct device::impl : public std::enable_shared_from_this }); } - array create_array(int count, common::bit_depth depth) + array create_array(int size) { - auto bytes_per_pixel = depth == common::bit_depth::bit8 ? 1 : 2; - auto buf = create_buffer(count * bytes_per_pixel, true); + auto buf = create_buffer(size, true); auto ptr = reinterpret_cast(buf->data()); return array(ptr, buf->size(), buf); } @@ -437,7 +436,7 @@ std::shared_ptr device::create_texture(int width, int height, int strid { return impl_->create_texture(width, height, stride, depth, true); } -array device::create_array(int size, common::bit_depth depth) { return impl_->create_array(size, depth); } +array device::create_array(int size) { return impl_->create_array(size); } std::future> device::copy_async(const array& source, int width, int height, int stride, common::bit_depth depth) { diff --git a/src/accelerator/ogl/util/device.h b/src/accelerator/ogl/util/device.h index e7c6982715..600ee2f374 100644 --- a/src/accelerator/ogl/util/device.h +++ b/src/accelerator/ogl/util/device.h @@ -47,7 +47,7 @@ class device final device& operator=(const device&) = delete; std::shared_ptr create_texture(int width, int height, int stride, common::bit_depth depth); - array create_array(int size, common::bit_depth depth); + array create_array(int size); std::future> copy_async(const array& source, int width, int height, int stride, common::bit_depth depth); From 1acf6bb213a897cc51e61fb20a9651ee7584114a Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 09:26:03 +0200 Subject: [PATCH 17/21] [FIXUP] Add comments related to paths supporting hdr frames in decklink consumer --- src/modules/decklink/consumer/frame.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/modules/decklink/consumer/frame.cpp b/src/modules/decklink/consumer/frame.cpp index f3e3f77570..b4a2384fe8 100644 --- a/src/modules/decklink/consumer/frame.cpp +++ b/src/modules/decklink/consumer/frame.cpp @@ -71,6 +71,7 @@ void convert_frame(const core::video_format_desc& channel_format_desc, // Fast path if (hdr) { + // Pack eight byte R16G16B16A16 pixels as four byte 10bit RGB R10G10B10XX const int NUM_THREADS = 4; auto rows_per_thread = decklink_format_desc.height / NUM_THREADS; size_t byte_count_line = get_row_bytes(decklink_format_desc, hdr); @@ -100,6 +101,8 @@ void convert_frame(const core::video_format_desc& channel_format_desc, } else { // Take a sub-region + // TODO: Add support for hdr frames + // Some repetetive numbers size_t byte_count_dest_line = (size_t)decklink_format_desc.width * 4; size_t byte_count_src_line = (size_t)channel_format_desc.width * 4; From e5f8bd8a856c7349c3b1b21ce8917d595ba5d23c Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 09:28:39 +0200 Subject: [PATCH 18/21] [FIXUP] Rename decklink producer param HDR to 10BIT --- src/modules/decklink/producer/decklink_producer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/decklink/producer/decklink_producer.cpp b/src/modules/decklink/producer/decklink_producer.cpp index 157ee85b5c..caa406a8cd 100644 --- a/src/modules/decklink/producer/decklink_producer.cpp +++ b/src/modules/decklink/producer/decklink_producer.cpp @@ -926,7 +926,7 @@ spl::shared_ptr create_producer(const core::frame_producer auto freeze_on_lost = contains_param(L"FREEZE_ON_LOST", params); - auto hdr = contains_param(L"HDR", params); + auto hdr = contains_param(L"10BIT", params); auto format_str = get_param(L"FORMAT", params); From bbda1bd31b2a4b50c30bdcc2a1988dd309ccf8d7 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 09:44:02 +0200 Subject: [PATCH 19/21] [FIXUP] Add bt601 as colorspace to decklink producer --- src/modules/decklink/producer/decklink_producer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/decklink/producer/decklink_producer.cpp b/src/modules/decklink/producer/decklink_producer.cpp index caa406a8cd..7d2fb56058 100644 --- a/src/modules/decklink/producer/decklink_producer.cpp +++ b/src/modules/decklink/producer/decklink_producer.cpp @@ -367,6 +367,8 @@ core::color_space get_color_space(IDeckLinkVideoInputFrame* video) if (SUCCEEDED(md->GetInt(bmdDeckLinkFrameMetadataColorspace, &color_space))) { if (color_space == bmdColorspaceRec2020) { return core::color_space::bt2020; + } else if (color_space == bmdColorspaceRec601) { + return core::color_space::bt601; } } } From c50120db0a1ffa70e8ae45c6af96907c87dac79b Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 09:44:27 +0200 Subject: [PATCH 20/21] [FIXUP] Add bt601 as possible colorspace to ffmpeg producer --- src/modules/ffmpeg/producer/av_producer.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/modules/ffmpeg/producer/av_producer.cpp b/src/modules/ffmpeg/producer/av_producer.cpp index ada52a5f6d..d70f50e521 100644 --- a/src/modules/ffmpeg/producer/av_producer.cpp +++ b/src/modules/ffmpeg/producer/av_producer.cpp @@ -77,8 +77,19 @@ struct Frame core::color_space get_color_space(const std::shared_ptr& video) { auto result = core::color_space::bt709; - if (video->colorspace == AVColorSpace::AVCOL_SPC_BT2020_NCL) - result = core::color_space::bt2020; + switch (video->colorspace) { + case AVColorSpace::AVCOL_SPC_BT2020_NCL: + result = core::color_space::bt2020; + break; + case AVColorSpace::AVCOL_SPC_BT470BG: + case AVColorSpace::AVCOL_SPC_SMPTE170M: + case AVColorSpace::AVCOL_SPC_SMPTE240M: + result = core::color_space::bt601; + break; + default: + break; + } + return result; } From b69d46eff5563a1a0669add8d0d4392829915751 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 6 May 2024 10:58:03 +0200 Subject: [PATCH 21/21] [FIXUP] Move default decklink hdr colorspace to config --- src/modules/decklink/consumer/config.cpp | 17 +++++++++++++++++ src/modules/decklink/consumer/config.h | 2 ++ .../decklink/consumer/decklink_consumer.cpp | 2 +- src/shell/casparcg.config | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/modules/decklink/consumer/config.cpp b/src/modules/decklink/consumer/config.cpp index 71a319e173..24e8626763 100644 --- a/src/modules/decklink/consumer/config.cpp +++ b/src/modules/decklink/consumer/config.cpp @@ -54,6 +54,19 @@ port_configuration parse_output_config(const boost::property_tree::wptree& ptre return port_config; } +core::color_space get_color_space(const std::wstring& str) +{ + auto color_space_str = boost::to_lower_copy(str); + if (color_space_str == L"bt709") + return core::color_space::bt709; + else if (color_space_str == L"bt2020") + return core::color_space::bt2020; + else if (color_space_str == L"bt601") + return core::color_space::bt601; + + CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Invalid decklink color-space, must be bt601, bt709 or bt2020")); +} + configuration parse_xml_config(const boost::property_tree::wptree& ptree, const core::video_format_repository& format_repository) { @@ -120,10 +133,14 @@ configuration parse_xml_config(const boost::property_tree::wptree& ptree, auto hdr_metadata = ptree.get_child_optional(L"hdr-metadata"); if(hdr_metadata) { + auto color_space = get_color_space(hdr_metadata->get(L"default-color-space", L"bt709")); + config.hdr_meta.min_dml = hdr_metadata->get(L"min-dml", config.hdr_meta.min_dml); config.hdr_meta.max_dml = hdr_metadata->get(L"max-dml", config.hdr_meta.max_dml); config.hdr_meta.max_fall = hdr_metadata->get(L"max-fall", config.hdr_meta.max_fall); config.hdr_meta.max_cll = hdr_metadata->get(L"max-cll", config.hdr_meta.max_cll); + + config.hdr_meta.default_color_space = color_space; } return config; diff --git a/src/modules/decklink/consumer/config.h b/src/modules/decklink/consumer/config.h index d6b97c7599..d11ae6c83f 100644 --- a/src/modules/decklink/consumer/config.h +++ b/src/modules/decklink/consumer/config.h @@ -21,6 +21,7 @@ #pragma once +#include #include namespace caspar { namespace decklink { @@ -50,6 +51,7 @@ struct hdr_meta_configuration float max_dml = 1000.0f; float max_fall = 100.0f; float max_cll = 1000.0f; + core::color_space default_color_space = core::color_space::bt709; }; struct configuration diff --git a/src/modules/decklink/consumer/decklink_consumer.cpp b/src/modules/decklink/consumer/decklink_consumer.cpp index b0d9200fac..2bd1aa4c91 100644 --- a/src/modules/decklink/consumer/decklink_consumer.cpp +++ b/src/modules/decklink/consumer/decklink_consumer.cpp @@ -710,7 +710,7 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback std::shared_ptr image_data = allocate_frame_data(decklink_format_desc_, config_.hdr); - schedule_next_video(image_data, nb_samples, video_scheduled_, core::color_space::bt2020); + schedule_next_video(image_data, nb_samples, video_scheduled_, config_.hdr_meta.default_color_space); for (auto& context : secondary_port_contexts_) { context->schedule_next_video(image_data, 0, video_scheduled_); } diff --git a/src/shell/casparcg.config b/src/shell/casparcg.config index a961b9d3a1..ee755b396e 100644 --- a/src/shell/casparcg.config +++ b/src/shell/casparcg.config @@ -104,6 +104,7 @@ 1000 [50..65535] 0.005 [0.0001-6.5535] 1000 [1-65535] + bt709 [bt601|bt709|bt2020] auto [auto|enable|disable]