diff --git a/build.gradle.kts b/build.gradle.kts index e8870e3..6c2519c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -21,16 +21,23 @@ dependencies { // With compose.desktop.common you will also lose @Preview functionality implementation(compose.desktop.currentOs) implementation(libs.compose.gl) + implementation(libs.compose.gl.natives) implementation(libs.jni.utils) implementation(libs.bundles.kotlinx) - implementation(libs.bundles.logging) + implementation(libs.slf4j.api) // for logging if (deployNative) { implementation(project(":native")) } implementation(kotlin("reflect")) +// implementation(libs.bundles.skiko) { +// version { +// strictly(libs.skiko.awt.runtime.linux.x64.get().version!!) +// } +// } testImplementation(libs.bundles.kotest) testImplementation(libs.mockk) + testImplementation(libs.logback.classic) } configurations.all { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 599ef8d..5c14454 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,9 @@ kotlinx-serialization = "1.7.1" kotlinx-datetime = "0.6.0" compose = "1.6.11" -compose-gl = "0.2.6" +compose-gl = "0.3.0" +jni-utils = "0.1.5" +skiko = "0.8.10-egl" slf4j = "2.0.13" logback = "1.5.6" @@ -15,7 +17,6 @@ kotest = "5.9.1" mockk = "1.13.12" idea-ext = "1.1.8" -jni-utils = "0.1.5" [libraries] kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } @@ -31,9 +32,13 @@ kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx- kotlinx-datetime = { group = "org.jetbrains.kotlinx", name = "kotlinx-datetime", version.ref = "kotlinx-datetime" } compose-gl = { group = "dev.silenium.compose.gl", name = "compose-gl", version.ref = "compose-gl" } +compose-gl-natives = { group = "dev.silenium.compose.gl", name = "compose-gl-natives-linux-x86_64", version.ref = "compose-gl" } ffmpeg-natives = { group = "dev.silenium.libs", name = "ffmpeg-natives", version = "7.0+0.2.0" } jni-utils = { group = "dev.silenium.libs.jni", name = "jni-utils", version.ref = "jni-utils" } +skiko-awt = { group = "org.jetbrains.skiko", name = "skiko-awt", version.ref = "skiko" } +skiko-awt-runtime-linux-x64 = { group = "org.jetbrains.skiko", name = "skiko-awt-runtime-linux-x64", version.ref = "skiko" } + slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" } logback-classic = { group = "ch.qos.logback", name = "logback-classic", version.ref = "logback" } @@ -64,14 +69,14 @@ kotlinx = [ "kotlinx-datetime", ] -logging = [ - "slf4j-api", - "logback-classic", -] - kotest = [ "kotest-runner-junit5", "kotest-property", "kotest-assertions-core", "kotest-assertions-json", ] + +skiko = [ + "skiko-awt", + "skiko-awt-runtime-linux-x64", +] diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 1378aff..10bc24d 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -60,9 +60,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") src/cpp/platform/linux/helper/EGL.cpp src/cpp/platform/linux/helper/EGL.hpp src/cpp/platform/linux/Errors.cpp - src/cpp/platform/linux/VAGLInteropImage.cpp - src/cpp/platform/linux/VAGLInteropImage.hpp - src/cpp/platform/linux/VAGLRenderInterop.cpp + src/cpp/platform/linux/VAEGLInteropImage.cpp + src/cpp/platform/linux/VAEGLInteropImage.hpp + src/cpp/platform/linux/VAGLXInteropImage.cpp + src/cpp/platform/linux/VAGLXInteropImage.hpp + src/cpp/platform/linux/VAGLXRenderInterop.cpp + src/cpp/platform/linux/VAEGLRenderInterop.cpp src/cpp/platform/linux/VaapiDecoder.cpp ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Windows") @@ -81,7 +84,7 @@ endif () if (CMAKE_SYSTEM_NAME STREQUAL "Linux") target_compile_definitions(${PROJECT_NAME} PRIVATE -D_LINUX) find_package(PkgConfig REQUIRED) - pkg_check_modules(GL REQUIRED IMPORTED_TARGET gl egl libva libva-drm libdrm) + pkg_check_modules(GL REQUIRED IMPORTED_TARGET gl egl libva libva-drm libdrm glx libva-glx) target_link_libraries(${PROJECT_NAME} PUBLIC PkgConfig::GL) target_include_directories(${PROJECT_NAME} PUBLIC "${JAVA_HOME}/include/linux") if (USE_SYSTEM_FFMPEG) diff --git a/native/src/cpp/helper/errors.cpp b/native/src/cpp/helper/errors.cpp index 6eeb1e4..6c9d88b 100644 --- a/native/src/cpp/helper/errors.cpp +++ b/native/src/cpp/helper/errors.cpp @@ -3,6 +3,7 @@ // #include "errors.hpp" +#include extern "C" { #include @@ -35,8 +36,7 @@ jobject avResultFailure(JNIEnv *env, const char *operation, const int returnCode const auto resultClass = env->FindClass("kotlin/Result$Failure"); const auto errorClass = env->FindClass("dev/silenium/compose/av/util/AVException"); const auto errorConstructor = env->GetMethodID(errorClass, "", "(Ljava/lang/String;I)V"); - const auto operationChars = env->NewStringUTF(operation); - const auto error = env->NewObject(errorClass, errorConstructor, operationChars, returnCode); + const auto error = env->NewObject(errorClass, errorConstructor, env->NewStringUTF(operation), returnCode); const auto resultConstructor = env->GetMethodID(resultClass, "", "(Ljava/lang/Throwable;)V"); const auto errorResult = env->NewObject(resultClass, resultConstructor, error); return errorResult; @@ -45,8 +45,14 @@ jobject avResultFailure(JNIEnv *env, const char *operation, const int returnCode jobject eglResultFailure(JNIEnv *env, const char *operation, const long returnCode) { const auto resultClass = env->FindClass("kotlin/Result$Failure"); const auto errorClass = env->FindClass("dev/silenium/compose/av/util/EGLException"); - const auto errorConstructor = env->GetMethodID(errorClass, "", "(Ljava/lang/String;I)V"); + const auto errorConstructor = env->GetMethodID(errorClass, "", "(Ljava/lang/String;J)V"); + std::cout << "errorConstructor: " << errorConstructor << std::endl; const auto error = env->NewObject(errorClass, errorConstructor, env->NewStringUTF(operation), returnCode); + std::cout << "error: " << error << std::endl; + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + } const auto resultConstructor = env->GetMethodID(resultClass, "", "(Ljava/lang/Throwable;)V"); const auto errorResult = env->NewObject(resultClass, resultConstructor, error); return errorResult; @@ -56,7 +62,13 @@ jobject glResultFailure(JNIEnv *env, const char *operation, const int returnCode const auto resultClass = env->FindClass("kotlin/Result$Failure"); const auto errorClass = env->FindClass("dev/silenium/compose/av/util/GLException"); const auto errorConstructor = env->GetMethodID(errorClass, "", "(Ljava/lang/String;I)V"); + std::cout << "errorConstructor: " << errorConstructor << std::endl; const auto error = env->NewObject(errorClass, errorConstructor, env->NewStringUTF(operation), returnCode); + std::cout << "error: " << error << std::endl; + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + } const auto resultConstructor = env->GetMethodID(resultClass, "", "(Ljava/lang/Throwable;)V"); const auto errorResult = env->NewObject(resultClass, resultConstructor, error); return errorResult; diff --git a/native/src/cpp/platform/linux/Errors.cpp b/native/src/cpp/platform/linux/Errors.cpp index 9419ce8..c8ac8d3 100644 --- a/native/src/cpp/platform/linux/Errors.cpp +++ b/native/src/cpp/platform/linux/Errors.cpp @@ -3,21 +3,21 @@ // #include "helper/EGL.hpp" -#include #include +#include +extern "C" { JNIEXPORT jstring JNICALL Java_dev_silenium_compose_av_util_ErrorsKt_eglErrorStringN( - JNIEnv *env, - jobject thiz, - const jlong error -) { + JNIEnv *env, + jobject thiz, + const jlong error) { return env->NewStringUTF(eglGetErrorString(error)); } JNIEXPORT jstring JNICALL Java_dev_silenium_compose_av_util_ErrorsKt_vaErrorStringN( - JNIEnv *env, - jobject thiz, - const jint error -) { + JNIEnv *env, + jobject thiz, + const jint error) { return env->NewStringUTF(vaErrorStr(error)); } +} diff --git a/native/src/cpp/platform/linux/VAGLInteropImage.cpp b/native/src/cpp/platform/linux/VAEGLInteropImage.cpp similarity index 73% rename from native/src/cpp/platform/linux/VAGLInteropImage.cpp rename to native/src/cpp/platform/linux/VAEGLInteropImage.cpp index 71949b1..9d0b0a9 100644 --- a/native/src/cpp/platform/linux/VAGLInteropImage.cpp +++ b/native/src/cpp/platform/linux/VAEGLInteropImage.cpp @@ -2,18 +2,18 @@ // Created by silenium-dev on 7/23/24. // -#include "VAGLInteropImage.hpp" +#include "VAEGLInteropImage.hpp" #include -VAGLInteropImage::VAGLInteropImage(const EGLDisplay display, +VAEGLInteropImage::VAEGLInteropImage(const EGLDisplay display, const std::vector &images, const std::vector &textures, const std::vector &swizzles) : eglDisplay(display), eglImages(images), textures(textures), swizzles(swizzles) { } -VAGLInteropImage::~VAGLInteropImage() { +VAEGLInteropImage::~VAEGLInteropImage() { const auto eglDestroyImageKHR = getFunc("eglDestroyImageKHR"); for (const auto &eglImage: eglImages) { if (eglImage != EGL_NO_IMAGE_KHR) { @@ -25,10 +25,10 @@ VAGLInteropImage::~VAGLInteropImage() { } } -const std::vector &VAGLInteropImage::planeTextures() const { +const std::vector &VAEGLInteropImage::planeTextures() const { return textures; } -const std::vector &VAGLInteropImage::planeSwizzles() const { +const std::vector &VAEGLInteropImage::planeSwizzles() const { return swizzles; } diff --git a/native/src/cpp/platform/linux/VAGLInteropImage.hpp b/native/src/cpp/platform/linux/VAEGLInteropImage.hpp similarity index 61% rename from native/src/cpp/platform/linux/VAGLInteropImage.hpp rename to native/src/cpp/platform/linux/VAEGLInteropImage.hpp index 9e8386c..db0ba6d 100644 --- a/native/src/cpp/platform/linux/VAGLInteropImage.hpp +++ b/native/src/cpp/platform/linux/VAEGLInteropImage.hpp @@ -9,21 +9,21 @@ #include "helper/EGL.hpp" #include -class VAGLInteropImage final : public GLInteropImage { +class VAEGLInteropImage final : public GLInteropImage { public: - VAGLInteropImage(EGLDisplay display, const std::vector &images, + VAEGLInteropImage(EGLDisplay display, const std::vector &images, const std::vector &textures, const std::vector &swizzles); - VAGLInteropImage(VAGLInteropImage &&) noexcept = default; + VAEGLInteropImage(VAEGLInteropImage &&) noexcept = default; - VAGLInteropImage &operator=(VAGLInteropImage &&) noexcept = default; + VAEGLInteropImage &operator=(VAEGLInteropImage &&) noexcept = default; - VAGLInteropImage(const VAGLInteropImage &) = delete; + VAEGLInteropImage(const VAEGLInteropImage &) = delete; - VAGLInteropImage &operator=(const VAGLInteropImage &) = delete; + VAEGLInteropImage &operator=(const VAEGLInteropImage &) = delete; - ~VAGLInteropImage() override; + ~VAEGLInteropImage() override; [[nodiscard]] const std::vector &planeTextures() const override; diff --git a/native/src/cpp/platform/linux/VAGLRenderInterop.cpp b/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp similarity index 96% rename from native/src/cpp/platform/linux/VAGLRenderInterop.cpp rename to native/src/cpp/platform/linux/VAEGLRenderInterop.cpp index 0dd04b8..1a2466b 100644 --- a/native/src/cpp/platform/linux/VAGLRenderInterop.cpp +++ b/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp @@ -2,12 +2,12 @@ // Created by silenium-dev on 7/15/24. // -#include "VAGLInteropImage.hpp" -#include "render/GLInteropImage.hpp" +#include "VAEGLInteropImage.hpp" #include "helper/errors.hpp" +#include "render/GLInteropImage.hpp" #include -#include +#include #include #include #include @@ -34,7 +34,7 @@ void closeDrm(const VADRMPRIMESurfaceDescriptor &drm) { extern "C" { JNIEXPORT jlong JNICALL -Java_dev_silenium_compose_av_platform_linux_VAGLRenderInteropKt_getVADisplayN( +Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_getVADisplayN( JNIEnv *env, jobject thiz, const jlong frame) { const auto avFrame = reinterpret_cast(frame); const auto deviceCtx = reinterpret_cast(avFrame->hw_frames_ctx->data)->device_ctx; @@ -54,7 +54,7 @@ std::map > > planeFractions{ }; JNIEXPORT jobject JNICALL -Java_dev_silenium_compose_av_platform_linux_VAGLRenderInteropKt_mapN(JNIEnv *env, jobject thiz, +Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *env, jobject thiz, const jint pixelFormat_, const jlong vaSurface_, const jlong vaDisplay_, const jlong eglDisplay_) { @@ -188,7 +188,7 @@ Java_dev_silenium_compose_av_platform_linux_VAGLRenderInteropKt_mapN(JNIEnv *env closeDrm(drm); glBindTexture(GL_TEXTURE_2D, prevTexture); - const auto interopImage = new VAGLInteropImage(eglDisplay, eglImages, textures, swizzles); + const auto interopImage = new VAEGLInteropImage(eglDisplay, eglImages, textures, swizzles); return resultSuccess(env, reinterpret_cast(interopImage)); } diff --git a/native/src/cpp/platform/linux/VAGLXInteropImage.cpp b/native/src/cpp/platform/linux/VAGLXInteropImage.cpp new file mode 100644 index 0000000..af7cca5 --- /dev/null +++ b/native/src/cpp/platform/linux/VAGLXInteropImage.cpp @@ -0,0 +1,33 @@ +// +// Created by silenium-dev on 7/23/24. +// + +#include "VAGLXInteropImage.hpp" + +#include +#include + +VAGLXInteropImage::VAGLXInteropImage( + VADisplay display, + VASurfaceID surface, + void *glxSurface, + unsigned int texture, + Swizzles swizzles) + : display(display), surface(surface), glxSurface(glxSurface), texture({texture}), swizzles({swizzles}) { +} + +VAGLXInteropImage::~VAGLXInteropImage() { + if (glxSurface != None) { + vaDestroySurfaceGLX(display, glxSurface); + } + vaDestroySurfaces(display, &surface, 1); + glDeleteTextures(1, &texture[0]); +} + +const std::vector &VAGLXInteropImage::planeTextures() const { + return texture; +} + +const std::vector &VAGLXInteropImage::planeSwizzles() const { + return swizzles; +} diff --git a/native/src/cpp/platform/linux/VAGLXInteropImage.hpp b/native/src/cpp/platform/linux/VAGLXInteropImage.hpp new file mode 100644 index 0000000..08f88cf --- /dev/null +++ b/native/src/cpp/platform/linux/VAGLXInteropImage.hpp @@ -0,0 +1,44 @@ +// +// Created by silenium-dev on 7/23/24. +// + +#ifndef VAINTEROPIMAGE_HPP +#define VAINTEROPIMAGE_HPP + +#include "helper/EGL.hpp" +#include "render/GLInteropImage.hpp" +#include + +class VAGLXInteropImage final : public GLInteropImage { +public: + VAGLXInteropImage( + VADisplay display, + VASurfaceID surface, + void *glxSurfaces, + unsigned int textures, + Swizzles swizzles); + + VAGLXInteropImage(VAGLXInteropImage &&) noexcept = default; + + VAGLXInteropImage &operator=(VAGLXInteropImage &&) noexcept = default; + + VAGLXInteropImage(const VAGLXInteropImage &) = delete; + + VAGLXInteropImage &operator=(const VAGLXInteropImage &) = delete; + + ~VAGLXInteropImage() override; + + [[nodiscard]] const std::vector &planeTextures() const override; + + [[nodiscard]] const std::vector &planeSwizzles() const override; + +private: + VADisplay display{nullptr}; + VASurfaceID surface{0}; + void *glxSurface{}; + std::vector texture{}; + std::vector swizzles{}; +}; + + +#endif//VAINTEROPIMAGE_HPP diff --git a/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp b/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp new file mode 100644 index 0000000..bcc7fb5 --- /dev/null +++ b/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp @@ -0,0 +1,286 @@ +// +// Created by silenium-dev on 7/15/24. +// + +#include "VAGLXInteropImage.hpp" +#include "helper/errors.hpp" +#include "render/GLInteropImage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +} + +// From ffmpeg: libavcodec/vaapi_decode.h +struct VAAPIDecodeContext { + VAConfigID va_config; + VAContextID va_context; + + AVHWDeviceContext *device; + AVVAAPIDeviceContext *hwctx; + + AVHWFramesContext *frames; + AVVAAPIFramesContext *hwfc; + + enum AVPixelFormat surface_format; + int surface_count; + + VASurfaceAttrib pixel_format_attribute; +}; + +// From ffmpeg: libavcodec/internal.h +struct AVCodecInternal { + /** + * When using frame-threaded decoding, this field is set for the first + * worker thread (e.g. to decode extradata just once). + */ + int is_copy; + + /** + * Audio encoders can set this flag during init to indicate that they + * want the small last frame to be padded to a multiple of pad_samples. + */ + int pad_samples; + + struct FramePool *pool; + + void *thread_ctx; + + /** + * This packet is used to hold the packet given to decoders + * implementing the .decode API; it is unused by the generic + * code for decoders implementing the .receive_frame API and + * may be freely used (but not freed) by them with the caveat + * that the packet will be unreferenced generically in + * avcodec_flush_buffers(). + */ + AVPacket *in_pkt; + struct AVBSFContext *bsf; + + /** + * Properties (timestamps+side data) extracted from the last packet passed + * for decoding. + */ + AVPacket *last_pkt_props; + + /** + * temporary buffer used for encoders to store their bitstream + */ + uint8_t *byte_buffer; + unsigned int byte_buffer_size; + + void *frame_thread_encoder; + + /** + * The input frame is stored here for encoders implementing the simple + * encode API. + * + * Not allocated in other cases. + */ + AVFrame *in_frame; + + /** + * When the AV_CODEC_FLAG_RECON_FRAME flag is used. the encoder should store + * here the reconstructed frame corresponding to the last returned packet. + * + * Not allocated in other cases. + */ + AVFrame *recon_frame; + + /** + * If this is set, then FFCodec->close (if existing) needs to be called + * for the parent AVCodecContext. + */ + int needs_close; + + /** + * Number of audio samples to skip at the start of the next decoded frame + */ + int skip_samples; + + /** + * hwaccel-specific private data + */ + void *hwaccel_priv_data; + + /** + * checks API usage: after codec draining, flush is required to resume operation + */ + int draining; + + /** + * Temporary buffers for newly received or not yet output packets/frames. + */ + AVPacket *buffer_pkt; + AVFrame *buffer_frame; + int draining_done; + +#if FF_API_DROPCHANGED + /* used when avctx flag AV_CODEC_FLAG_DROPCHANGED is set */ + int changed_frames_dropped; + int initial_format; + int initial_width, initial_height; + int initial_sample_rate; + AVChannelLayout initial_ch_layout; +#endif + +#if CONFIG_LCMS2 + FFIccContext icc; /* used to read and write embedded ICC profiles */ +#endif + + /** + * Set when the user has been warned about a failed allocation from + * a fixed frame pool. + */ + int warned_on_failed_allocation_from_fixed_pool; +}; + +extern "C" { +JNIEXPORT jlong JNICALL +Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_getVADisplayN( + JNIEnv *env, jobject thiz, const jlong frame) { + const auto avFrame = reinterpret_cast(frame); + const auto deviceCtx = reinterpret_cast(avFrame->hw_frames_ctx->data)->device_ctx; + const auto vaContext = static_cast(deviceCtx->hwctx); + return reinterpret_cast(vaContext->display); +} + +JNIEXPORT jobject JNICALL +Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_mapN(JNIEnv *env, jobject thiz, + const jlong vaSurface_, const jlong vaDisplay_, + const jlong frame_, const jlong codecContext_) { + const auto vaDisplay = reinterpret_cast(vaDisplay_); + const auto vaSurface = static_cast(vaSurface_); + std::cout << "VA Surface: " << vaSurface << std::endl; + const auto frame = reinterpret_cast(frame_); + const auto codecContext = reinterpret_cast(codecContext_); + const auto vaapiContext = reinterpret_cast(codecContext->internal->hwaccel_priv_data); + std::cout << "Priv Data: " << vaapiContext << std::endl; + std::cout << "VA Context: " << vaapiContext->va_context << std::endl; + std::cout << "VA surface_count: " << vaapiContext->surface_count << std::endl; + std::cout << "VA pixel_format_attribute: " << vaapiContext->pixel_format_attribute.value.value.i << std::endl; + // std::cout << "VASurface: " << vaSurface << std::endl; + // std::cout << "VADisplay: " << vaDisplay << std::endl; + + std::vector + rgbAttribs{ + { + .type = VASurfaceAttribPixelFormat, + .flags = VA_SURFACE_ATTRIB_SETTABLE, + .value = { + .type = VAGenericValueTypeInteger, + .value = {.i = VA_FOURCC_BGRX}, + }, + }, + }; + + VAImage vaImage{}; + auto ret = vaDeriveImage(vaDisplay, vaSurface, &vaImage); + if (ret != VA_STATUS_SUCCESS) { + return vaResultFailure(env, "vaDeriveImage", ret); + } + std::cout << "VA Image: " << vaImage.image_id << std::endl; + const auto width = vaImage.width; + const auto height = vaImage.height; + const auto num_planes = vaImage.num_planes; + unsigned int pitches[3]; + unsigned int offsets[3]; + memcpy(pitches, vaImage.pitches, sizeof(pitches)); + memcpy(offsets, vaImage.offsets, sizeof(offsets)); + vaDestroyImage(vaDisplay, vaImage.image_id); + + vaSyncSurface(vaDisplay, vaSurface); + std::cout << "Width: " << width << std::endl; + std::cout << "Height: " << height << std::endl; + + const auto alignedWidth = width + (16 - width % 16); + const auto alignedHeight = height + (16 - height % 16); + VASurfaceID rgbSurface{}; + ret = vaCreateSurfaces(vaDisplay, VA_RT_FORMAT_RGB32, alignedWidth, alignedHeight, &rgbSurface, 1, rgbAttribs.data(), rgbAttribs.size()); + if (ret != VA_STATUS_SUCCESS) { + return vaResultFailure(env, "vaCreateSurfaces", ret); + } + std::cout << "RGB Surface: " << rgbSurface << std::endl; + + VAContextID vaContextID{}; + ret = vaCreateContext(vaDisplay, vaapiContext->va_config, width, height, VA_PROGRESSIVE, &rgbSurface, 1, &vaContextID); + if (ret != VA_STATUS_SUCCESS) { + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaCreateContext", ret); + } + std::cout << "VA Context ID: " << vaContextID << std::endl; + + ret = vaBeginPicture(vaDisplay, vaContextID, rgbSurface); + if (ret != VA_STATUS_SUCCESS) { + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaBeginPicture", ret); + } + std::cout << "Begin Picture" << std::endl; + VABufferID vaBuffer{}; + ret = vaCreateBuffer(vaDisplay, vaContextID, VABufferType::VAProcPipelineParameterBufferType, sizeof(VAProcPipelineParameterBuffer), 1, nullptr, &vaBuffer); + if (ret != VA_STATUS_SUCCESS) { + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaCreateBuffer", ret); + } + std::cout << "Create Buffer" << std::endl; + VAProcPipelineParameterBuffer *pipelineParameterBuffer; + ret = vaMapBuffer(vaDisplay, vaBuffer, reinterpret_cast(&pipelineParameterBuffer)); + if (ret != VA_STATUS_SUCCESS) { + vaDestroyBuffer(vaDisplay, vaBuffer); + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaMapBuffer", ret); + } + pipelineParameterBuffer->surface = vaSurface; + pipelineParameterBuffer->surface_region = nullptr; + pipelineParameterBuffer->output_region = nullptr; + pipelineParameterBuffer->filter_flags = VA_FILTER_SCALING_FAST; + pipelineParameterBuffer->filters = nullptr; + vaUnmapBuffer(vaDisplay, vaBuffer); + std::cout << "Configured Buffer" << std::endl; + + ret = vaRenderPicture(vaDisplay, vaContextID, &vaBuffer, 1); + if (ret != VA_STATUS_SUCCESS) { + vaDestroyBuffer(vaDisplay, vaBuffer); + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaRenderPicture", ret); + } + std::cout << "Render Picture" << std::endl; + ret = vaEndPicture(vaDisplay, vaContextID); + if (ret != VA_STATUS_SUCCESS) { + vaDestroyBuffer(vaDisplay, vaBuffer); + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaEndPicture", ret); + } + std::cout << "End Picture" << std::endl; + vaDestroyBuffer(vaDisplay, vaBuffer); + vaDestroyContext(vaDisplay, vaContextID); + + void *glxSurface{}; + GLuint texture{}; + glGenTextures(1, &texture); + ret = vaCreateSurfaceGLX(vaDisplay, GL_TEXTURE_2D, texture, &glxSurface); + if (ret != VA_STATUS_SUCCESS) { + vaDestroySurfaces(vaDisplay, &rgbSurface, 1); + return vaResultFailure(env, "vaCreateSurfaceGLX", ret); + } + + const auto interopImage = new VAGLXInteropImage(vaDisplay, rgbSurface, glxSurface, texture, Swizzles::Identity); + + return resultSuccess(env, reinterpret_cast(interopImage)); +} +} diff --git a/native/src/cpp/render/Swizzles.hpp b/native/src/cpp/render/Swizzles.hpp index 5a2cfc5..3448d4d 100644 --- a/native/src/cpp/render/Swizzles.hpp +++ b/native/src/cpp/render/Swizzles.hpp @@ -17,6 +17,15 @@ struct Swizzles { Swizzle g = Swizzle::USE_GREEN; Swizzle b = Swizzle::USE_BLUE; Swizzle a = Swizzle::USE_ALPHA; + + static Swizzles Identity; +}; + +Swizzles Swizzles::Identity{ + .r = Swizzle::USE_RED, + .g = Swizzle::USE_GREEN, + .b = Swizzle::USE_BLUE, + .a = Swizzle::USE_ALPHA, }; #endif //SWIZZLES_HPP diff --git a/native/src/cpp/util/Errors.cpp b/native/src/cpp/util/Errors.cpp index 226fa2a..7a4a1b6 100644 --- a/native/src/cpp/util/Errors.cpp +++ b/native/src/cpp/util/Errors.cpp @@ -2,6 +2,7 @@ // Created by silenium-dev on 7/21/24. // +#include #include extern "C" { @@ -16,4 +17,31 @@ JNIEXPORT jstring JNICALL Java_dev_silenium_compose_av_util_ErrorsKt_avErrorStri av_make_error_string(errorStr, AV_ERROR_MAX_STRING_SIZE, error); return env->NewStringUTF(errorStr); } + +JNIEXPORT jstring JNICALL Java_dev_silenium_compose_av_util_ErrorsKt_glErrorStringN( + JNIEnv *env, + jobject thiz, + const jint error +) { + switch (error) { + case GL_NO_ERROR: + return env->NewStringUTF("GL_NO_ERROR"); + case GL_INVALID_ENUM: + return env->NewStringUTF("GL_INVALID_ENUM"); + case GL_INVALID_VALUE: + return env->NewStringUTF("GL_INVALID_VALUE"); + case GL_INVALID_OPERATION: + return env->NewStringUTF("GL_INVALID_OPERATION"); + case GL_INVALID_FRAMEBUFFER_OPERATION: + return env->NewStringUTF("GL_INVALID_FRAMEBUFFER_OPERATION"); + case GL_OUT_OF_MEMORY: + return env->NewStringUTF("GL_OUT_OF_MEMORY"); + case GL_STACK_UNDERFLOW: + return env->NewStringUTF("GL_STACK_UNDERFLOW"); + case GL_STACK_OVERFLOW: + return env->NewStringUTF("GL_STACK_OVERFLOW"); + default: + return env->NewStringUTF("Unknown"); + } +} } diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt similarity index 98% rename from src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLRenderInterop.kt rename to src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt index 8f319ad..364ad27 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLRenderInterop.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt @@ -7,7 +7,7 @@ import dev.silenium.compose.av.render.GLRenderInterop import dev.silenium.compose.av.util.Natives import org.lwjgl.egl.EGL15 -class VAGLRenderInterop( +class VAEGLRenderInterop( override val decoder: VaapiDecoder, private val eglDisplay: Long = EGL15.eglGetCurrentDisplay(), ) : GLRenderInterop { diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt new file mode 100644 index 0000000..49c0c94 --- /dev/null +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt @@ -0,0 +1,42 @@ +package dev.silenium.compose.av.platform.linux + +import dev.silenium.compose.av.data.AVPixelFormat +import dev.silenium.compose.av.data.Frame +import dev.silenium.compose.av.render.GLInteropImage +import dev.silenium.compose.av.render.GLRenderInterop +import dev.silenium.compose.av.util.Natives +import org.slf4j.LoggerFactory + +class VAGLXRenderInterop(override val decoder: VaapiDecoder) : GLRenderInterop { + init { + Natives.ensureLoaded() + log.error("VAGLXRenderInterop is not working yet") + } + + override fun isSupported(frame: Frame): Boolean { + return frame.isHW && frame.format == AVPixelFormat.AV_PIX_FMT_VAAPI + } + + override fun map(frame: Frame): Result { + val vaDisplay = getVADisplayN(frame.nativePointer.address) + val vaSurface = frame.rawData[3] + return mapN( + vaSurface, + vaDisplay, + frame.nativePointer.address, + decoder.nativePointer.address + ).map { GLInteropImage(frame, it) } + } + + companion object { + private val log = LoggerFactory.getLogger(VAGLXRenderInterop::class.java) + } +} + +private external fun getVADisplayN(frame: Long): Long +private external fun mapN( + vaSurface: Long, + vaDisplay: Long, + frame: Long, + codecContext: Long, +): Result diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt index b8243a3..dafc77b 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt @@ -6,14 +6,14 @@ import dev.silenium.compose.av.decode.Decoder import dev.silenium.compose.av.demux.Stream import dev.silenium.compose.av.util.Natives -class VaapiDecoder(stream: Stream, vaDevice: String) : Decoder(stream) { +class VaapiDecoder(stream: Stream, val vaDevice: String) : Decoder(stream) { override val nativePointer: NativePointer = createDecoderN(stream.nativePointer.address, vaDevice).getOrThrow() .asNativePointer(::releaseDecoder) val vaDisplay by lazy { getVADisplayN(nativePointer.address) } - override fun createGLRenderInterop() = VAGLRenderInterop(this) + override fun createGLRenderInterop() = VAGLXRenderInterop(this) companion object { init { diff --git a/src/main/kotlin/dev/silenium/compose/av/player/Player.kt b/src/main/kotlin/dev/silenium/compose/av/player/Player.kt index 373116d..f34f95c 100644 --- a/src/main/kotlin/dev/silenium/compose/av/player/Player.kt +++ b/src/main/kotlin/dev/silenium/compose/av/player/Player.kt @@ -25,7 +25,7 @@ import dev.silenium.compose.gl.surface.rememberGLSurfaceState import kotlinx.coroutines.* import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.channels.Channel -import org.lwjgl.opengles.GLES30.* +import org.lwjgl.opengl.GL30.* import java.nio.file.Path import kotlin.time.Duration.Companion.milliseconds diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml new file mode 100644 index 0000000..74599de --- /dev/null +++ b/src/test/resources/logback.xml @@ -0,0 +1,10 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + +