From 7bc073249f33372a62a9417f1ee121d35773627d Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 24 May 2026 12:08:57 -0500 Subject: [PATCH] =?UTF-8?q?qt+renderer/vulkan:=20dmabuf=20flows=20end-to-e?= =?UTF-8?q?nd=20(libghostty=20=E2=86=92=20Qt=20widget)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Vulkan renderer now produces real dmabuf frames and the Qt-side GhosttySurface mmaps them straight into a QImage for display. Three plumbing gaps closed; the placeholder is now backed by actual GPU output (currently a clear-color frame since `RenderPass.step` is still a stub, but it's a real frame from the GPU). The three fixes: 1. `vulkan/Frame.complete` now calls `self.target.present()` at the end (after the fence wait). `opengl/Frame.zig`'s complete does the same — invokes `api.present(target)` — but the Vulkan version was missing it, so libghostty rendered frames into the dmabuf and then... never told the host. Adding the `target.present()` call routes the rendered fd through the `ghostty_platform_vulkan_s.present` callback to the apprt. 2. Qt-side `syncSurfaceSize` was early-exiting on `makeCurrent()` failure (always true on the Vulkan path since there's no GL context). That meant `ghostty_surface_set_size` never fired, libghostty thought the surface was 0x0, and the renderer skipped every frame. Branched on `m_useVulkan` so the Vulkan path skips the FBO bookkeeping but still propagates size + DPR and kicks `renderTerminal()` for the first frame. 3. `GhosttySurface::renderTerminal` for the Vulkan path now just calls `ghostty_surface_draw(m_surface)` and lets the platform's `present` callback machinery wire the result back. The OpenGL path's GL context + FBO bookkeeping is skipped — libghostty owns its own target VkImage. Qt-side dmabuf import: - New `GhosttySurface::presentVulkanDmabuf` (Q_INVOKABLE) is the apprt-side entry point for the platform's `present` callback. `mmap()`s the dmabuf fd (LINEAR tiling means the bytes are directly readable as BGRA), copies into a QImage, schedules a paint on the GUI thread via `QMetaObject::invokeMethod`. - `vulkan::Host::cbPresent` no longer just logs — it now dispatches to `vulkan::presentToGhosttySurface` which casts the userdata back to a `GhosttySurface *` and forwards the parameters. - `paintEvent` keeps the placeholder when `m_image.isNull()` (i.e. before the first frame lands) and falls through to the same QImage blit the OpenGL path uses once a frame arrives. Userdata routing: `Host::asPlatform(surface_userdata)` now actually uses its argument — every `GhosttySurface` constructs its `ghostty_platform_vulkan_s` with `this` as userdata, so the `present` callback can identify which surface a dmabuf is for. The handle-lookup callbacks (instance/physicalDevice/device/queue) ignore the userdata and route through `Host::instance()` since there's only one process-wide Vulkan setup. Verified output of `GHASTTY_RENDERER=vulkan ghastty-vulkan`: [vulkan] device ready: NVIDIA GeForce RTX 2080 (Vulkan 1.4.329, qfi=0) [ghastty] Vulkan.beginFrame: first call, target 800x600 [ghastty] first Vulkan frame: 800x600 stride=3200 fourcc=0x34325241 - stride 3200 = 800 * 4 (linear-packed BGRA, no padding). - fourcc 0x34325241 = 'AR24' = DRM_FORMAT_ARGB8888 (correct mapping for our VK_FORMAT_B8G8R8A8_UNORM target). - The window now displays the actual rendered dmabuf — currently just the clear color from `RenderPass.begin`'s CLEAR loadOp, but it's GPU-rendered content reaching the window. Co-Authored-By: claude-flow --- qt/src/GhosttySurface.cpp | 129 +++++++++++++++++++++++++++++++--- qt/src/GhosttySurface.h | 23 ++++-- qt/src/vulkan/Host.cpp | 71 ++++++++++++------- qt/src/vulkan/Host.h | 8 ++- src/renderer/Vulkan.zig | 22 ++++++ src/renderer/vulkan/Frame.zig | 7 ++ 6 files changed, 220 insertions(+), 40 deletions(-) diff --git a/qt/src/GhosttySurface.cpp b/qt/src/GhosttySurface.cpp index af20fce5e..dede217a2 100644 --- a/qt/src/GhosttySurface.cpp +++ b/qt/src/GhosttySurface.cpp @@ -11,12 +11,15 @@ #include "vulkan/Host.h" #include +#include #include #include #include #include #include +#include + #include #include #include @@ -189,6 +192,18 @@ void GhosttySurface::syncSurfaceSize() { m_fbh = h; m_fbDpr = dpr; + // Vulkan path: libghostty manages the target image itself (it + // allocates the dmabuf-exportable VkImage). We just need to tell + // it the new pixel size + DPR and kick a first render — same + // shape as the OpenGL path below, minus the FBO bookkeeping. + if (m_useVulkan) { + ghostty_surface_set_content_scale(m_surface, dpr, dpr); + ghostty_surface_set_size(m_surface, static_cast(w), + static_cast(h)); + renderTerminal(); + return; + } + if (!makeCurrent()) return; delete m_fbo; QOpenGLFramebufferObjectFormat fmt; @@ -302,7 +317,18 @@ void GhosttySurface::flashScrollbar() { } void GhosttySurface::renderTerminal() { - if (!m_surface || !m_fbo || !makeCurrent()) return; + if (!m_surface) return; + + // Vulkan path: libghostty owns its target VkImage; it renders into + // it directly and presents via the apprt dmabuf callback. No GL + // context, no FBO, no readback — just kick the draw and let the + // platform-side `present` machinery wire the result back to us. + if (m_useVulkan) { + ghostty_surface_draw(m_surface); + return; + } + + if (!m_fbo || !makeCurrent()) return; // libghostty renders into its own target and blits the result to the // currently bound framebuffer — bind ours so we get the final image. @@ -325,19 +351,18 @@ void GhosttySurface::renderTerminal() { } void GhosttySurface::paintEvent(QPaintEvent *) { - // Vulkan-backed surface: libghostty hands frames to the host via - // a dmabuf fd; we don't yet composite them back into this widget. - // Paint a visible placeholder so the (translucent) MainWindow - // isn't completely invisible. Replace with the imported - // QRhiTexture once the dmabuf-import path lands. - if (m_useVulkan) { + // Vulkan-backed surface, no frame imported yet: paint a visible + // placeholder so the (translucent) MainWindow isn't completely + // invisible. Once `presentVulkanDmabuf` lands a frame, fall + // through to the regular blit path below. + if (m_useVulkan && m_image.isNull()) { QPainter painter(this); painter.setCompositionMode(QPainter::CompositionMode_Source); painter.fillRect(rect(), QColor(40, 22, 56)); // muted purple — debug placeholder painter.setPen(QColor(220, 220, 220)); painter.drawText(rect(), Qt::AlignCenter, - QStringLiteral("Vulkan renderer\n(dmabuf import not yet wired)")); + QStringLiteral("Vulkan renderer\n(awaiting first dmabuf frame)")); paintResizeOverlay(painter); return; } @@ -1243,3 +1268,91 @@ void GhosttySurface::glReleaseCurrent(void *) { void GhosttySurface::glPresent(void *) { // No-op: the frame is read back from the framebuffer, not swapped. } + +// --- libghostty Vulkan present path ---------------------------------- + +void GhosttySurface::presentVulkanDmabuf( + int dmabuf_fd, + quint32 drm_format, + quint64 drm_modifier, + quint32 width, + quint32 height, + quint32 stride) { + // Called from the renderer thread. We mmap the dmabuf, copy the + // bytes into a QImage, and hand the QImage to the GUI thread for + // paint via `QMetaObject::invokeMethod`. The fd is a borrow (per + // the `ghostty_platform_vulkan_s` contract); libghostty closes it + // when the underlying memory is freed. + (void)drm_modifier; // LINEAR for v1; not used here. + + // First-frame breadcrumb so we know the dmabuf hand-off is firing. + static bool first_frame = true; + if (first_frame) { + first_frame = false; + std::fprintf(stderr, + "[ghastty] first Vulkan frame: %ux%u stride=%u fourcc=0x%08x\n", + width, height, stride, drm_format); + } + + // sanity check the size before we allocate / mmap. + if (dmabuf_fd < 0 || width == 0 || height == 0 || stride < width * 4) + return; + + const size_t bytes = static_cast(stride) * height; + void *mapped = ::mmap(nullptr, bytes, PROT_READ, MAP_SHARED, dmabuf_fd, 0); + if (mapped == MAP_FAILED) { + std::fprintf(stderr, "[ghastty] mmap of dmabuf fd=%d failed: %s\n", + dmabuf_fd, std::strerror(errno)); + return; + } + // QImage holds the pixel data by copying when constructed with + // `Format_ARGB32` from a buffer with explicit stride. We then + // detach (copy()) so the QImage survives the unmap. + // + // drm_format ARGB8888 (0x34325241 = "AR24") matches QImage's + // Format_ARGB32 byte order on little-endian (B,G,R,A in memory). + // We unconditionally use ARGB32 here because the renderer currently + // emits BGRA only — extend with a format switch when other formats + // come online. + (void)drm_format; + const QImage stamped( + static_cast(mapped), + static_cast(width), + static_cast(height), + static_cast(stride), + QImage::Format_ARGB32); + QImage owned = stamped.copy(); + ::munmap(mapped, bytes); + + // Marshal to the GUI thread. The lambda captures `owned` by value. + QPointer selfp(this); + QMetaObject::invokeMethod( + this, + [selfp, owned]() mutable { + if (!selfp) return; + selfp->m_image = std::move(owned); + selfp->update(); + }, + Qt::QueuedConnection); +} + +// Trampoline so `Host.cpp` doesn't need to include the full +// `GhosttySurface.h`. The forward declaration lives in +// `vulkan/Host.cpp` (namespace scope, not anonymous, so the linker +// resolves this definition). +namespace vulkan { + +void presentToGhosttySurface( + void *surface, + int dmabuf_fd, + uint32_t drm_format, + uint64_t drm_modifier, + uint32_t width, + uint32_t height, + uint32_t stride) { + if (surface == nullptr) return; + static_cast(surface)->presentVulkanDmabuf( + dmabuf_fd, drm_format, drm_modifier, width, height, stride); +} + +} // namespace vulkan diff --git a/qt/src/GhosttySurface.h b/qt/src/GhosttySurface.h index 4b7b0e843..753093119 100644 --- a/qt/src/GhosttySurface.h +++ b/qt/src/GhosttySurface.h @@ -143,6 +143,21 @@ public: void setPwd(const QString &pwd); const QString &pwd() const { return m_pwd; } + // Apprt-side entry point for the Vulkan `present` callback. + // libghostty hands us a dmabuf fd pointing at the rendered + // VkImage's memory; we mmap it (LINEAR tiling means the bytes + // are directly readable as BGRA), copy the pixels into a QImage, + // and schedule a repaint. Thread-safe: the callback fires from + // the renderer thread; the QImage handoff goes through + // `QMetaObject::invokeMethod` to the GUI thread. + Q_INVOKABLE void presentVulkanDmabuf( + int dmabuf_fd, + quint32 drm_format, + quint64 drm_modifier, + quint32 width, + quint32 height, + quint32 stride); + protected: bool event(QEvent *) override; void paintEvent(QPaintEvent *) override; @@ -216,10 +231,10 @@ private: QImage m_image; // last frame, read back from m_fbo // True when this surface is using the Vulkan platform. The - // paintEvent uses this to draw a visible placeholder until the - // host-side dmabuf-import + composite work lands; otherwise the - // widget would paint nothing on a translucent window and look - // invisible. + // paintEvent uses this to draw a visible placeholder when no + // dmabuf has been imported yet; once + // `presentVulkanDmabuf` has filled `m_image` the placeholder + // gives way to the actual rendered content. bool m_useVulkan = false; // GL objects for the alpha-premultiply pass. diff --git a/qt/src/vulkan/Host.cpp b/qt/src/vulkan/Host.cpp index 5752e7cad..3d591ee1b 100644 --- a/qt/src/vulkan/Host.cpp +++ b/qt/src/vulkan/Host.cpp @@ -11,6 +11,19 @@ namespace vulkan { +// Forward declaration of the entry point in `GhosttySurface.cpp` that +// receives a presented frame. Declared here at namespace scope (not +// in the anonymous namespace below) so its external definition in +// the other TU resolves at link time. +void presentToGhosttySurface( + void *surface, + int dmabuf_fd, + uint32_t drm_format, + uint64_t drm_modifier, + uint32_t width, + uint32_t height, + uint32_t stride); + namespace { constexpr const char *kRequiredDeviceExtensions[] = { @@ -51,39 +64,49 @@ std::optional findGraphicsQueueFamily(VkPhysicalDevice pd) { // ---- Platform callback trampolines ---------------------------------- // -// `ghostty_platform_vulkan_s` is a plain C ABI; the callback -// signatures take a `void *userdata` that libghostty hands back to -// each callback. We use that as our `Host *`. +// `ghostty_platform_vulkan_s` is a plain C ABI; the callback signatures +// take a `void *userdata` that libghostty hands back to each callback. +// The handle-lookup callbacks (instance / physical_device / device / +// queue / queue_family_index / get_instance_proc_addr) ignore the +// userdata and resolve through the process singleton — there's only +// one Vulkan setup per process. The `present` callback DOES use the +// userdata: it's the `GhosttySurface *` that owns the rendered +// target, so we can hand the dmabuf back to the right widget. void *cbGetInstanceProcAddr(void *ud, const char *name) { - auto *self = static_cast(ud); - // Cast through `void(*)()` to silence strict-aliasing concerns - // about converting a function pointer to `void *` (the ABI we - // exposed in include/ghostty.h returns `void *` for portability, - // matching the OpenGL `get_proc_address` callback shape). - auto fp = vkGetInstanceProcAddr(self->vkInstance(), name); + (void)ud; + auto *host = Host::instance(); + if (host == nullptr) return nullptr; + auto fp = vkGetInstanceProcAddr(host->vkInstance(), name); return reinterpret_cast(fp); } void *cbInstance(void *ud) { - return static_cast(ud)->vkInstance(); + (void)ud; + auto *host = Host::instance(); + return host != nullptr ? host->vkInstance() : nullptr; } void *cbPhysicalDevice(void *ud) { - return static_cast(ud)->vkPhysicalDevice(); + (void)ud; + auto *host = Host::instance(); + return host != nullptr ? host->vkPhysicalDevice() : nullptr; } void *cbDevice(void *ud) { - return static_cast(ud)->vkDevice(); + (void)ud; + auto *host = Host::instance(); + return host != nullptr ? host->vkDevice() : nullptr; } void *cbQueue(void *ud) { - return static_cast(ud)->vkQueue(); + (void)ud; + auto *host = Host::instance(); + return host != nullptr ? host->vkQueue() : nullptr; } uint32_t cbQueueFamilyIndex(void *ud) { - return static_cast(ud)->vkQueueFamilyIndex(); + (void)ud; + auto *host = Host::instance(); + return host != nullptr ? host->vkQueueFamilyIndex() : 0; } -// Present: libghostty hands us the rendered frame as a dmabuf fd. -// For now this just logs — actual import + display via QRhiTexture -// is the next chunk of Qt-side work. void cbPresent( void *ud, int dmabuf_fd, @@ -92,12 +115,9 @@ void cbPresent( uint32_t width, uint32_t height, uint32_t stride) { - (void)ud; - std::fprintf( - stderr, - "[vulkan] present cb: fd=%d fourcc=0x%08x mod=0x%016lx %ux%u stride=%u\n", - dmabuf_fd, drm_format, static_cast(drm_modifier), - width, height, stride); + if (ud == nullptr) return; + ::vulkan::presentToGhosttySurface(ud, dmabuf_fd, drm_format, + drm_modifier, width, height, stride); } } // namespace @@ -188,10 +208,9 @@ Host::~Host() { if (m_instance != VK_NULL_HANDLE) vkDestroyInstance(m_instance, nullptr); } -ghostty_platform_vulkan_s Host::asPlatform(void *userdata) const { - (void)userdata; +ghostty_platform_vulkan_s Host::asPlatform(void *surface_userdata) const { ghostty_platform_vulkan_s p{}; - p.userdata = const_cast(this); + p.userdata = surface_userdata; p.get_instance_proc_addr = cbGetInstanceProcAddr; p.instance = cbInstance; p.physical_device = cbPhysicalDevice; diff --git a/qt/src/vulkan/Host.h b/qt/src/vulkan/Host.h index f1ed974da..c0161ca20 100644 --- a/qt/src/vulkan/Host.h +++ b/qt/src/vulkan/Host.h @@ -35,8 +35,12 @@ public: static Host *instance(); /// Build a `ghostty_platform_vulkan_s` callback struct populated - /// with this host's handles. Pass to `ghostty_surface_config_s`. - ghostty_platform_vulkan_s asPlatform(void *userdata) const; + /// with this host's handles. `surface_userdata` is round-tripped + /// through as the `userdata` field — used by the `present` + /// callback to identify which `GhosttySurface` the dmabuf is for. + /// The other handle-lookup callbacks ignore it and route through + /// `Host::instance()`. + ghostty_platform_vulkan_s asPlatform(void *surface_userdata) const; VkInstance vkInstance() const { return m_instance; } VkPhysicalDevice vkPhysicalDevice() const { return m_physicalDevice; } diff --git a/src/renderer/Vulkan.zig b/src/renderer/Vulkan.zig index 8c9d5afd0..c6820f33d 100644 --- a/src/renderer/Vulkan.zig +++ b/src/renderer/Vulkan.zig @@ -260,6 +260,18 @@ pub fn surfaceSize(self: *const Vulkan) !struct { width: u32, height: u32 } { pub fn present(self: *Vulkan, target: Target) !void { _ = self; + // Breadcrumb for the bring-up — flag the first present so we can + // tell from logs whether the frame loop is actually firing. + const first_present = struct { + var yes: bool = true; + }; + if (first_present.yes) { + first_present.yes = false; + std.debug.print( + "[ghastty] Vulkan.present: first frame, fd={} stride={} {}x{}\n", + .{ target.fd, target.stride, target.width, target.height }, + ); + } // The target is already populated by the time we get here: // `Frame.complete` ended the command buffer, submitted with the // fence, and waited for the GPU to finish before returning. So @@ -281,6 +293,16 @@ pub fn beginFrame( target: *Target, ) !Frame { _ = renderer; + // Breadcrumb so we can see in logs when the renderer actually + // starts a frame (which calls our beginFrame). One-shot per + // process to avoid spamming. + const first_begin = struct { + var yes: bool = true; + }; + if (first_begin.yes) { + first_begin.yes = false; + std.debug.print("[ghastty] Vulkan.beginFrame: first call, target {}x{}\n", .{ target.width, target.height }); + } const dev = devicePtr(); // Lazy per-thread resource init. The first call to `beginFrame` diff --git a/src/renderer/vulkan/Frame.zig b/src/renderer/vulkan/Frame.zig index 495ae133e..75094a588 100644 --- a/src/renderer/vulkan/Frame.zig +++ b/src/renderer/vulkan/Frame.zig @@ -146,6 +146,13 @@ pub fn complete(self: *const Self, sync: bool) void { log.err("vkWaitForFences (frame) failed: result={}", .{r}); } } + + // Hand the rendered target off to the host. This mirrors what + // `opengl/Frame.zig`'s `complete` does at the same point: it + // calls `self.renderer.api.present(self.target.*)`. Our analog + // is `Target.present()`, which routes through the platform's + // `present` callback (the apprt-side dmabuf consumer). + self.target.present(); } /// Begin a render pass recording into this frame's command buffer.