Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix multi-GPU support for Nvidia #147

Merged
merged 8 commits into from
Mar 2, 2025
Merged

Fix multi-GPU support for Nvidia #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
86e36b5
allocator: Don't force linear for multigpu if unsupported
PlasmaPower Feb 24, 2025
847fe2e
drm: Use an intermediate CPU buffer for blit if necessary
PlasmaPower Feb 24, 2025
8ce5a7c
drm: Add comment and cleanup style a bit
PlasmaPower Feb 24, 2025
96cbe55
drm: Fix formatting from comment addition
PlasmaPower Feb 24, 2025
b7f01bf
drm: Use std::vector and std::span for intermediateBuf
PlasmaPower Feb 25, 2025
bcbe886
drm: Fix style and add null check
PlasmaPower Feb 26, 2025
9347746
drm: Remove TODO about pixel buf format
PlasmaPower Feb 26, 2025
d9b0a7a
Merge branch 'main' into nvidia-multi-gpu
PlasmaPower Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/allocator/GBM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,16 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
return;
}

// FIXME: Nvidia cannot render to linear buffers. What do?
if (MULTIGPU) {
allocator->backend->log(AQ_LOG_DEBUG, "GBM: Buffer is marked as multigpu, forcing linear");
explicitModifiers = {DRM_FORMAT_MOD_LINEAR};
// Try to use the linear format if available for cross-GPU compatibility.
// However, Nvidia doesn't support linear, so this is a best-effort basis.
for (auto const& f : FORMATS) {
if (f.drmFormat == DRM_FORMAT_MOD_LINEAR) {
allocator->backend->log(AQ_LOG_DEBUG, "GBM: Buffer is marked as multigpu, using linear format");
explicitModifiers = {DRM_FORMAT_MOD_LINEAR};
break;
}
}
}

uint32_t flags = GBM_BO_USE_RENDERING;
Expand Down
14 changes: 10 additions & 4 deletions src/backend/drm/DRM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1699,9 +1699,12 @@ bool Aquamarine::CDRMOutput::commitState(bool onlyTest) {
return false;
}

auto NEWAQBUF = mgpu.swapchain->next(nullptr);
auto NEWAQBUF = mgpu.swapchain->next(nullptr);
SP<Aquamarine::CDRMRenderer> primaryRenderer;
if (backend->primary)
primaryRenderer = backend->primary->rendererState.renderer;
auto blitResult = backend->rendererState.renderer->blit(
STATE.buffer, NEWAQBUF, (COMMITTED & COutputState::eOutputStateProperties::AQ_OUTPUT_STATE_EXPLICIT_IN_FENCE) ? STATE.explicitInFence : -1);
STATE.buffer, NEWAQBUF, primaryRenderer, (COMMITTED & COutputState::eOutputStateProperties::AQ_OUTPUT_STATE_EXPLICIT_IN_FENCE) ? STATE.explicitInFence : -1);
if (!blitResult.success) {
backend->backend->log(AQ_LOG_ERROR, "drm: Backend requires blit, but blit failed");
return false;
Expand Down Expand Up @@ -1881,8 +1884,11 @@ bool Aquamarine::CDRMOutput::setCursor(SP<IBuffer> buffer, const Vector2D& hotsp
return false;
}

auto NEWAQBUF = mgpu.cursorSwapchain->next(nullptr);
if (!backend->rendererState.renderer->blit(buffer, NEWAQBUF).success) {
auto NEWAQBUF = mgpu.cursorSwapchain->next(nullptr);
SP<Aquamarine::CDRMRenderer> primaryRenderer;
if (backend->primary)
primaryRenderer = backend->primary->rendererState.renderer;
if (!backend->rendererState.renderer->blit(buffer, NEWAQBUF, primaryRenderer).success) {
backend->backend->log(AQ_LOG_ERROR, "drm: Backend requires blit, but cursor blit failed");
return false;
}
Expand Down
84 changes: 74 additions & 10 deletions src/backend/drm/Renderer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ void CDRMRenderer::loadEGLAPI() {
loadGLProc(&proc.eglWaitSyncKHR, "eglWaitSyncKHR");
loadGLProc(&proc.eglCreateSyncKHR, "eglCreateSyncKHR");
loadGLProc(&proc.eglDupNativeFenceFDANDROID, "eglDupNativeFenceFDANDROID");
loadGLProc(&proc.glEGLImageTargetRenderbufferStorageOES, "glEGLImageTargetRenderbufferStorageOES");
loadGLProc(&proc.glReadnPixelsEXT, "glReadnPixelsEXT");

if (EGLEXTENSIONS.contains("EGL_EXT_device_base") || EGLEXTENSIONS.contains("EGL_EXT_device_enumeration"))
loadGLProc(&proc.eglQueryDevicesEXT, "eglQueryDevicesEXT");
Expand Down Expand Up @@ -694,6 +694,49 @@ SGLTex CDRMRenderer::glTex(Hyprutils::Memory::CSharedPointer<IBuffer> buffa) {
return tex;
}

constexpr GLenum PIXEL_BUFFER_FORMAT = GL_RGBA;

void CDRMRenderer::readBuffer(Hyprutils::Memory::CSharedPointer<IBuffer> buf, std::span<uint8_t> out) {
CEglContextGuard eglContext(*this);
auto hadAttachment = buf->attachments.get(AQ_ATTACHMENT_DRM_RENDERER_DATA);
auto att = (CDRMRendererBufferAttachment*)hadAttachment.get();
if (!hadAttachment) {
// should never remove anything, but JIC. We'll leak an EGLImage if this removes anything.
buf->attachments.removeByType(AQ_ATTACHMENT_DRM_RENDERER_DATA);
auto newAttachment = makeShared<CDRMRendererBufferAttachment>(self, buf, nullptr, 0, 0, SGLTex{}, std::vector<uint8_t>());
att = newAttachment.get();
buf->attachments.add(newAttachment);
}

auto dma = buf->dmabuf();
if (!att->eglImage) {
att->eglImage = createEGLImage(dma);
if (att->eglImage == EGL_NO_IMAGE_KHR) {
backend->log(AQ_LOG_ERROR, std::format("EGL (readBuffer): createEGLImage failed: {}", eglGetError()));
return;
}

GLCALL(glGenRenderbuffers(1, &att->rbo));
GLCALL(glBindRenderbuffer(GL_RENDERBUFFER, att->rbo));
GLCALL(proc.glEGLImageTargetRenderbufferStorageOES(GL_RENDERBUFFER, (GLeglImageOES)att->eglImage));
GLCALL(glBindRenderbuffer(GL_RENDERBUFFER, 0));

GLCALL(glGenFramebuffers(1, &att->fbo));
GLCALL(glBindFramebuffer(GL_FRAMEBUFFER, att->fbo));
GLCALL(glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, att->rbo));

if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
backend->log(AQ_LOG_ERROR, std::format("EGL (readBuffer): glCheckFramebufferStatus failed: {}", glGetError()));
return;
}
}

GLCALL(glBindFramebuffer(GL_FRAMEBUFFER, att->fbo));
GLCALL(proc.glReadnPixelsEXT(0, 0, dma.size.x, dma.size.y, GL_RGBA, GL_UNSIGNED_BYTE, out.size(), out.data()));

GLCALL(glBindFramebuffer(GL_FRAMEBUFFER, 0));
}

inline const float fullVerts[] = {
1, 0, // top right
0, 0, // top left
Expand Down Expand Up @@ -821,7 +864,7 @@ void CDRMRenderer::clearBuffer(IBuffer* buf) {
proc.eglDestroyImageKHR(egl.display, rboImage);
}

CDRMRenderer::SBlitResult CDRMRenderer::blit(SP<IBuffer> from, SP<IBuffer> to, int waitFD) {
CDRMRenderer::SBlitResult CDRMRenderer::blit(SP<IBuffer> from, SP<IBuffer> to, SP<CDRMRenderer> primaryRenderer, int waitFD) {
CEglContextGuard eglContext(*this);

if (from->dmabuf().size != to->dmabuf().size) {
Expand All @@ -839,22 +882,40 @@ CDRMRenderer::SBlitResult CDRMRenderer::blit(SP<IBuffer> from, SP<IBuffer> to, i
// both from and to have the same AQ_ATTACHMENT_DRM_RENDERER_DATA.
// Those buffers always come from different swapchains, so it's OK.

SGLTex fromTex;
SGLTex fromTex;
auto fromDma = from->dmabuf();
std::span<uint8_t> intermediateBuf;
{
auto attachment = from->attachments.get(AQ_ATTACHMENT_DRM_RENDERER_DATA);
if (attachment) {
TRACE(backend->log(AQ_LOG_TRACE, "EGL (blit): From attachment found"));
auto att = (CDRMRendererBufferAttachment*)attachment.get();
fromTex = att->tex;
auto att = (CDRMRendererBufferAttachment*)attachment.get();
fromTex = att->tex;
intermediateBuf = att->intermediateBuf;
}

if (!fromTex.image) {
if (!fromTex.image && intermediateBuf.empty()) {
backend->log(AQ_LOG_DEBUG, "EGL (blit): No attachment in from, creating a new image");
fromTex = glTex(from);

auto newAttachment = makeShared<CDRMRendererBufferAttachment>(self, from, nullptr, 0, 0, fromTex, std::vector<uint8_t>());
if (!fromTex.image && primaryRenderer) {
backend->log(AQ_LOG_DEBUG, "EGL (blit): Failed to create image from source buffer directly, allocating intermediate buffer");
static_assert(PIXEL_BUFFER_FORMAT == GL_RGBA); // If the pixel buffer format changes, the below size calculation probably needs to as well.
newAttachment->intermediateBuf.resize(fromDma.size.x * fromDma.size.y * 4);
intermediateBuf = newAttachment->intermediateBuf;
fromTex.target = GL_TEXTURE_2D;
GLCALL(glGenTextures(1, &fromTex.texid));
}

// should never remove anything, but JIC. We'll leak an EGLImage if this removes anything.
from->attachments.removeByType(AQ_ATTACHMENT_DRM_RENDERER_DATA);
from->attachments.add(makeShared<CDRMRendererBufferAttachment>(self, from, nullptr, 0, 0, fromTex));
from->attachments.add(newAttachment);
}

if (!intermediateBuf.empty() && primaryRenderer) {
// Note: this might modify from's attachments
primaryRenderer->readBuffer(from, intermediateBuf);
}
}

Expand Down Expand Up @@ -909,7 +970,7 @@ CDRMRenderer::SBlitResult CDRMRenderer::blit(SP<IBuffer> from, SP<IBuffer> to, i

// should never remove anything, but JIC. We'll leak an RBO and FBO if this removes anything.
to->attachments.removeByType(AQ_ATTACHMENT_DRM_RENDERER_DATA);
to->attachments.add(makeShared<CDRMRendererBufferAttachment>(self, to, rboImage, fboID, rboID, SGLTex{}));
to->attachments.add(makeShared<CDRMRendererBufferAttachment>(self, to, rboImage, fboID, rboID, SGLTex{}, std::vector<uint8_t>()));
}
}

Expand Down Expand Up @@ -957,6 +1018,9 @@ CDRMRenderer::SBlitResult CDRMRenderer::blit(SP<IBuffer> from, SP<IBuffer> to, i
GLCALL(glTexParameteri(fromTex.target, GL_TEXTURE_MAG_FILTER, GL_NEAREST));
GLCALL(glTexParameteri(fromTex.target, GL_TEXTURE_MIN_FILTER, GL_NEAREST));

if (!intermediateBuf.empty())
GLCALL(glTexImage2D(fromTex.target, 0, PIXEL_BUFFER_FORMAT, fromDma.size.x, fromDma.size.y, 0, PIXEL_BUFFER_FORMAT, GL_UNSIGNED_BYTE, intermediateBuf.data()));

GLCALL(glUseProgram(SHADER.program));
GLCALL(glDisable(GL_BLEND));
GLCALL(glDisable(GL_SCISSOR_TEST));
Expand Down Expand Up @@ -1032,7 +1096,7 @@ bool CDRMRenderer::verifyDestinationDMABUF(const SDMABUFAttrs& attrs) {
}

CDRMRendererBufferAttachment::CDRMRendererBufferAttachment(Hyprutils::Memory::CWeakPointer<CDRMRenderer> renderer_, Hyprutils::Memory::CSharedPointer<IBuffer> buffer,
EGLImageKHR image, GLuint fbo_, GLuint rbo_, SGLTex tex_) :
eglImage(image), fbo(fbo_), rbo(rbo_), tex(tex_), renderer(renderer_) {
EGLImageKHR image, GLuint fbo_, GLuint rbo_, SGLTex tex_, std::vector<uint8_t> intermediateBuf_) :
eglImage(image), fbo(fbo_), rbo(rbo_), tex(tex_), intermediateBuf(intermediateBuf_), renderer(renderer_) {
bufferDestroy = buffer->events.destroy.registerListener([this](std::any d) { renderer->onBufferAttachmentDrop(this); });
}
9 changes: 7 additions & 2 deletions src/backend/drm/Renderer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include <optional>
#include <tuple>
#include <vector>
#include <span>

namespace Aquamarine {

Expand All @@ -24,7 +25,7 @@ namespace Aquamarine {
class CDRMRendererBufferAttachment : public IAttachment {
public:
CDRMRendererBufferAttachment(Hyprutils::Memory::CWeakPointer<CDRMRenderer> renderer_, Hyprutils::Memory::CSharedPointer<IBuffer> buffer, EGLImageKHR image, GLuint fbo_,
GLuint rbo_, SGLTex tex);
GLuint rbo_, SGLTex tex, std::vector<uint8_t> intermediateBuf_);
virtual ~CDRMRendererBufferAttachment() {
;
}
Expand All @@ -36,6 +37,7 @@ namespace Aquamarine {
GLuint fbo = 0, rbo = 0;
SGLTex tex;
Hyprutils::Signal::CHyprSignalListener bufferDestroy;
std::vector<uint8_t> intermediateBuf;

Hyprutils::Memory::CWeakPointer<CDRMRenderer> renderer;
};
Expand Down Expand Up @@ -78,7 +80,8 @@ namespace Aquamarine {
std::optional<int> syncFD;
};

SBlitResult blit(Hyprutils::Memory::CSharedPointer<IBuffer> from, Hyprutils::Memory::CSharedPointer<IBuffer> to, int waitFD = -1);
SBlitResult blit(Hyprutils::Memory::CSharedPointer<IBuffer> from, Hyprutils::Memory::CSharedPointer<IBuffer> to,
Hyprutils::Memory::CSharedPointer<CDRMRenderer> primaryRenderer, int waitFD = -1);
// can't be a SP<> because we call it from buf's ctor...
void clearBuffer(IBuffer* buf);

Expand Down Expand Up @@ -106,6 +109,7 @@ namespace Aquamarine {
PFNEGLDEBUGMESSAGECONTROLKHRPROC eglDebugMessageControlKHR = nullptr;
PFNEGLQUERYDEVICESEXTPROC eglQueryDevicesEXT = nullptr;
PFNEGLQUERYDEVICESTRINGEXTPROC eglQueryDeviceStringEXT = nullptr;
PFNGLREADNPIXELSEXTPROC glReadnPixelsEXT = nullptr;
} proc;

struct {
Expand All @@ -128,6 +132,7 @@ namespace Aquamarine {
} egl;

SGLTex glTex(Hyprutils::Memory::CSharedPointer<IBuffer> buf);
void readBuffer(Hyprutils::Memory::CSharedPointer<IBuffer> buf, std::span<uint8_t> out);

Hyprutils::Memory::CWeakPointer<CDRMRenderer> self;
std::vector<SGLFormat> formats;
Expand Down