Skip to content

Commit

Permalink
encode: add compute image pre-processor
Browse files Browse the repository at this point in the history
  • Loading branch information
@zlatinski
zlatinski committed Dec 17, 2024
1 parent 0a7ebcd commit a6990d1
Show file tree
Hide file tree
Showing 7 changed files with 153 additions and 36 deletions.
6 changes: 3 additions & 3 deletions vk_video_encoder/demos/vk-video-enc/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ set(sources
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceContext.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceContext.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanShaderCompiler.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanShaderCompiler.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceMemoryImpl.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceMemoryImpl.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanShaderCompiler.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VkBufferResource.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VkBufferResource.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VkImageResource.cpp
Expand All @@ -67,8 +67,6 @@ set(sources
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDescriptorSetLayout.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanCommandBuffersSet.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanCommandBuffersSet.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSession.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSession.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSessionParameters.cpp
Expand All @@ -84,6 +82,8 @@ set(sources
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/pattern.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoUtils.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoUtils.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilter.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilter.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilterYuvCompute.cpp
Expand Down
21 changes: 17 additions & 4 deletions vk_video_encoder/demos/vk-video-enc/Main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,11 @@ int main(int argc, char** argv)
}
}

VkQueueFlags requestVideoComputeQueueMask = 0;
if (encoderConfig->enablePreprocessComputeFilter == VK_TRUE) {
requestVideoComputeQueueMask = VK_QUEUE_COMPUTE_BIT;
}

VkSharedBaseObj<VulkanVideoDisplayQueue<VulkanEncoderInputFrame>> videoDispayQueue;
result = CreateVulkanVideoEncodeDisplayQueue(&vkDevCtxt,
encoderConfig->encodeWidth,
Expand Down Expand Up @@ -180,7 +185,10 @@ int main(int argc, char** argv)
}

result = vkDevCtxt.InitPhysicalDevice(encoderConfig->deviceId, encoderConfig->GetDeviceUUID(),
(VK_QUEUE_GRAPHICS_BIT | requestVideoDecodeQueueMask | requestVideoEncodeQueueMask),
(VK_QUEUE_GRAPHICS_BIT |
requestVideoComputeQueueMask |
requestVideoDecodeQueueMask |
requestVideoEncodeQueueMask),
displayShell,
requestVideoDecodeQueueMask,
(VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR |
Expand All @@ -203,7 +211,8 @@ int main(int argc, char** argv)
false, // createTransferQueue
true, // createGraphicsQueue
true, // createDisplayQueue
(encoderConfig->selectVideoWithComputeQueue == 1) // createComputeQueue
((encoderConfig->selectVideoWithComputeQueue == 1) || // createComputeQueue
(encoderConfig->enablePreprocessComputeFilter == VK_TRUE))
);
if (result != VK_SUCCESS) {

Expand All @@ -227,7 +236,10 @@ int main(int argc, char** argv)

// No display presentation and no decoder - just the encoder
result = vkDevCtxt.InitPhysicalDevice(encoderConfig->deviceId, encoderConfig->GetDeviceUUID(),
(requestVideoDecodeQueueMask | requestVideoEncodeQueueMask | VK_QUEUE_TRANSFER_BIT),
(requestVideoComputeQueueMask |
requestVideoDecodeQueueMask |
requestVideoEncodeQueueMask |
VK_QUEUE_TRANSFER_BIT),
nullptr,
requestVideoDecodeQueueMask,
(VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR |
Expand All @@ -251,7 +263,8 @@ int main(int argc, char** argv)
((vkDevCtxt.GetVideoEncodeQueueFlag() & VK_QUEUE_TRANSFER_BIT) == 0), // createTransferQueue
false, // createGraphicsQueue
false, // createDisplayQueue
(encoderConfig->selectVideoWithComputeQueue == 1) // createComputeQueue
((encoderConfig->selectVideoWithComputeQueue == 1) || // createComputeQueue
(encoderConfig->enablePreprocessComputeFilter == VK_TRUE))
);
if (result != VK_SUCCESS) {

Expand Down
13 changes: 13 additions & 0 deletions vk_video_encoder/libs/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ set(LIBVKVIDEOENCODER
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/HelpersDispatchTable.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceContext.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceContext.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanShaderCompiler.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanShaderCompiler.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceMemoryImpl.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanDeviceMemoryImpl.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VkBufferResource.cpp
Expand All @@ -99,6 +101,12 @@ set(LIBVKVIDEOENCODER
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSessionParameters.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/pattern.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/pattern.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanComputePipeline.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilter.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilter.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilterYuvCompute.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFilterYuvCompute.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanSamplerYcbcrConversion.cpp
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanSamplerYcbcrConversion.h
${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanFenceSet.cpp
Expand All @@ -115,8 +123,11 @@ include_directories(BEFORE "${CMAKE_CURRENT_LIST_DIR}/../")
include_directories(BEFORE ${VULKAN_VIDEO_ENCODER_INCLUDE}/../libs)
include_directories(BEFORE ${VULKAN_VIDEO_ENCODER_INCLUDE})
include_directories(BEFORE ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT})
include_directories(BEFORE ${SHADERC_ROOT_PATH}/install/include)

add_library(${VULKAN_VIDEO_ENCODER_LIB} SHARED ${LIBVKVIDEOENCODER})
# Link the libraries
target_link_libraries(${VULKAN_VIDEO_ENCODER_LIB} PUBLIC ${GLSLANG_LIBRARIES})
# Ensure the library depends on the generation of these files
add_dependencies(${VULKAN_VIDEO_ENCODER_LIB} GenerateDispatchTables)

Expand All @@ -141,6 +152,8 @@ if(WIN32)
endif()

add_library(${VULKAN_VIDEO_ENCODER_STATIC_LIB} STATIC ${LIBVKVIDEOENCODER})
# Link the libraries
target_link_libraries(${VULKAN_VIDEO_ENCODER_STATIC_LIB} PUBLIC ${GLSLANG_LIBRARIES})
# Ensure the library depends on the generation of these files
add_dependencies(${VULKAN_VIDEO_ENCODER_STATIC_LIB} GenerateDispatchTables)
target_include_directories(${VULKAN_VIDEO_ENCODER_STATIC_LIB} PUBLIC ${VULKAN_VIDEO_ENCODER_INCLUDE} ${VULKAN_VIDEO_ENCODER_INCLUDE}/../NvVideoParser PRIVATE include)
Expand Down
7 changes: 7 additions & 0 deletions vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "VkVideoEncoder/VkVideoGopStructure.h"
#include "VkVideoCore/VkVideoCoreProfile.h"
#include "VkVideoCore/VulkanVideoCapabilities.h"
#include "VkCodecUtils/VulkanFilterYuvCompute.h"

struct EncoderConfigH264;
struct EncoderConfigH265;
Expand Down Expand Up @@ -731,6 +732,9 @@ struct EncoderConfig : public VkVideoRefCountBase {
EncoderInputFileHandler inputFileHandler;
EncoderOutputFileHandler outputFileHandler;
EncoderQpMapFileHandler qpMapFileHandler;

VulkanFilterYuvCompute::FilterType filterType;

uint32_t validate : 1;
uint32_t validateVerbose : 1;
uint32_t verbose : 1;
Expand All @@ -741,6 +745,7 @@ struct EncoderConfig : public VkVideoRefCountBase {
uint32_t enableVideoDecoder : 1;
uint32_t enableHwLoadBalancing : 1;
uint32_t selectVideoWithComputeQueue : 1;
uint32_t enablePreprocessComputeFilter : 1;
uint32_t enableOutOfOrderRecording : 1; // Testing only - don't use for production!

EncoderConfig()
Expand Down Expand Up @@ -815,6 +820,7 @@ struct EncoderConfig : public VkVideoRefCountBase {
, max_dec_frame_buffering()
, chroma_sample_loc_type()
, inputFileHandler()
, filterType(VulkanFilterYuvCompute::YCBCRCOPY)
, validate(false)
, validateVerbose(false)
, verbose(false)
Expand All @@ -826,6 +832,7 @@ struct EncoderConfig : public VkVideoRefCountBase {
, enableHwLoadBalancing(false)
, selectVideoWithComputeQueue(false)
, enableOutOfOrderRecording(false)
, enablePreprocessComputeFilter(false)
{ }

virtual ~EncoderConfig() {}
Expand Down
128 changes: 101 additions & 27 deletions vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ VkResult VkVideoEncoder::StageInputFrame(VkSharedBaseObj<VkVideoEncodeFrameInfo>
assert(encodeFrameInfo);

if (encodeFrameInfo->srcEncodeImageResource == nullptr) {

bool success = m_inputImagePool->GetAvailableImage(encodeFrameInfo->srcEncodeImageResource,
VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR);
assert(success);
Expand All @@ -329,19 +330,33 @@ VkResult VkVideoEncoder::StageInputFrame(VkSharedBaseObj<VkVideoEncodeFrameInfo>
VkSharedBaseObj<VkImageResourceView> srcEncodeImageView;
encodeFrameInfo->srcEncodeImageResource->GetImageView(srcEncodeImageView);

VkImageLayout linearImgNewLayout = TransitionImageLayout(cmdBuf, linearInputImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
VkImageLayout srcImgNewLayout = TransitionImageLayout(cmdBuf, srcEncodeImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
(void)linearImgNewLayout;
(void)srcImgNewLayout;
VkResult result;
if (m_inputComputeFilter == nullptr) {
VkImageLayout linearImgNewLayout = TransitionImageLayout(cmdBuf, linearInputImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
VkImageLayout srcImgNewLayout = TransitionImageLayout(cmdBuf, srcEncodeImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
(void)linearImgNewLayout;
(void)srcImgNewLayout;

VkExtent2D copyImageExtent {
std::min(m_encoderConfig->encodeWidth, m_encoderConfig->input.width),
std::min(m_encoderConfig->encodeHeight, m_encoderConfig->input.height)
};

VkExtent2D copyImageExtent {
std::min(m_encoderConfig->encodeWidth, m_encoderConfig->input.width),
std::min(m_encoderConfig->encodeHeight, m_encoderConfig->input.height)
};
CopyLinearToOptimalImage(cmdBuf, linearInputImageView, srcEncodeImageView, copyImageExtent);

CopyLinearToOptimalImage(cmdBuf, linearInputImageView, srcEncodeImageView, copyImageExtent);
} else {

result = m_inputComputeFilter->RecordCommandBuffer(cmdBuf,
linearInputImageView,
encodeFrameInfo->srcStagingImageView->GetPictureResourceInfo(),
srcEncodeImageView,
encodeFrameInfo->srcEncodeImageResource->GetPictureResourceInfo(),
encodeFrameInfo->inputCmdBuffer->GetNodePoolIndex());
if (result != VK_SUCCESS) {
return result;
}
}

VkResult result;
// Stage QPMap if it needs staging. Reuse the same command buffer used for staging of the input image
if (m_encoderConfig->enableQpMap && (m_qpMapTiling != VK_IMAGE_TILING_LINEAR)) {
result = StageInputFrameQpMap(encodeFrameInfo, cmdBuf);
Expand Down Expand Up @@ -416,8 +431,11 @@ VkResult VkVideoEncoder::SubmitStagedInputFrame(VkSharedBaseObj<VkVideoEncodeFra

VkFence queueCompleteFence = encodeFrameInfo->inputCmdBuffer->GetFence();
assert(VK_NOT_READY == m_vkDevCtx->GetFenceStatus(*m_vkDevCtx, queueCompleteFence));
VkResult result = m_vkDevCtx->MultiThreadedQueueSubmit(((m_vkDevCtx->GetVideoEncodeQueueFlag() & VK_QUEUE_TRANSFER_BIT) != 0) ?
VulkanDeviceContext::ENCODE : VulkanDeviceContext::TRANSFER,
const VulkanDeviceContext::QueueFamilySubmitType submitType =
(m_inputComputeFilter != nullptr) ? VulkanDeviceContext::COMPUTE :
(((m_vkDevCtx->GetVideoEncodeQueueFlag() & VK_QUEUE_TRANSFER_BIT) != 0) ?
VulkanDeviceContext::ENCODE : VulkanDeviceContext::TRANSFER);
VkResult result = m_vkDevCtx->MultiThreadedQueueSubmit(submitType,
0, 1, &submitInfo,
queueCompleteFence);

Expand Down Expand Up @@ -917,23 +935,78 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj<EncoderConfig>& encoderConf
}
}

result = VulkanCommandBufferPool::Create(m_vkDevCtx, m_inputCommandBufferPool);
if(result != VK_SUCCESS) {
fprintf(stderr, "\nInitEncoder Error: Failed to create m_inputCommandBufferPool.\n");
return result;
if (encoderConfig->enablePreprocessComputeFilter) {

const VkSamplerYcbcrRange ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; // FIXME
const VkSamplerYcbcrModelConversion ycbcrModelConversion = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020; // FIXME
const YcbcrPrimariesConstants ycbcrPrimariesConstants = GetYcbcrPrimariesConstants(YcbcrBtStandardBt2020); // FIXME

const VkSamplerYcbcrConversionCreateInfo ycbcrConversionCreateInfo {
VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO,
nullptr,
m_imageInFormat,
ycbcrModelConversion,
ycbcrRange,
{ VK_COMPONENT_SWIZZLE_IDENTITY,
VK_COMPONENT_SWIZZLE_IDENTITY,
VK_COMPONENT_SWIZZLE_IDENTITY,
VK_COMPONENT_SWIZZLE_IDENTITY
},
VK_CHROMA_LOCATION_MIDPOINT, // FIXME
VK_CHROMA_LOCATION_MIDPOINT, // FIXME
VK_FILTER_LINEAR,
false
};

static const VkSamplerCreateInfo samplerInfo = {
VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
nullptr,
0,
VK_FILTER_LINEAR, VK_FILTER_LINEAR, VK_SAMPLER_MIPMAP_MODE_NEAREST,
VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
// mipLodBias anisotropyEnable maxAnisotropy compareEnable compareOp minLod maxLod borderColor
// unnormalizedCoordinates
0.0, false, 0.00, false, VK_COMPARE_OP_NEVER, 0.0, 16.0, VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE, false
};

result = VulkanFilterYuvCompute::Create(m_vkDevCtx,
m_vkDevCtx->GetComputeQueueFamilyIdx(),
0, // queueIndex
encoderConfig->filterType,
encoderConfig->numInputImages,
m_imageInFormat, // in filter format (can be RGB)
m_imageInFormat, // out filter - same as input for now.
&ycbcrConversionCreateInfo,
&ycbcrPrimariesConstants,
&samplerInfo,
m_inputComputeFilter);
}

result = m_inputCommandBufferPool->Configure( m_vkDevCtx,
encoderConfig->numInputImages, // numPoolNodes
((m_vkDevCtx->GetVideoEncodeQueueFlag() & VK_QUEUE_TRANSFER_BIT) != 0) ?
m_vkDevCtx->GetVideoEncodeQueueFamilyIdx() :
m_vkDevCtx->GetTransferQueueFamilyIdx(), // queueFamilyIndex
false, // createQueryPool - not needed for the input transfer
nullptr, // pVideoProfile - not needed for the input transfer
true, // createSemaphores
true // createFences
);
if(result != VK_SUCCESS) {
if ((result == VK_SUCCESS) && (m_inputComputeFilter != nullptr) ) {

m_inputCommandBufferPool = m_inputComputeFilter;

} else {

result = VulkanCommandBufferPool::Create(m_vkDevCtx, m_inputCommandBufferPool);
if(result != VK_SUCCESS) {
fprintf(stderr, "\nInitEncoder Error: Failed to create m_inputCommandBufferPool.\n");
return result;
}

result = m_inputCommandBufferPool->Configure( m_vkDevCtx,
encoderConfig->numInputImages, // numPoolNodes
((m_vkDevCtx->GetVideoEncodeQueueFlag() & VK_QUEUE_TRANSFER_BIT) != 0) ?
m_vkDevCtx->GetVideoEncodeQueueFamilyIdx() :
m_vkDevCtx->GetTransferQueueFamilyIdx(), // queueFamilyIndex
false, // createQueryPool - not needed for the input transfer
nullptr, // pVideoProfile - not needed for the input transfer
true, // createSemaphores
true // createFences
);
}

if (result != VK_SUCCESS) {
fprintf(stderr, "\nInitEncoder Error: Failed to Configure m_inputCommandBufferPool.\n");
return result;
}
Expand Down Expand Up @@ -1626,6 +1699,7 @@ int32_t VkVideoEncoder::DeinitEncoder()
m_inputImagePool = nullptr;
m_dpbImagePool = nullptr;

m_inputComputeFilter = nullptr;
m_inputCommandBufferPool = nullptr;
m_encodeCommandBufferPool = nullptr;

Expand Down
Loading

0 comments on commit a6990d1

Please sign in to comment.