input: use std.Io.Writer for key encoder, new API, expose via libghostty (#9030)

This modernizes `KeyEncoder` to a new `std.Io.Writer`-based API.
Additionally, instead of a single struct, it is now an `encode` function
that takes a series of more focused options. This is more idiomatic Zig
while also making it easier to expose via libghostty-vt.

libghostty-vt Zig module also gains access to key encoding APIs. The C
APIs will follow another time.

Converting the KeyEncoder tests was done using AI:
https://ampcode.com/threads/T-9731bbdc-e0a9-41ad-9404-2b781a66ee39
Reviewed and understood.

src/Surface.zig

@@ -271,7 +271,7 @@ const DerivedConfig = struct {
     mouse_scroll_multiplier: configpkg.MouseScrollMultiplier,
     mouse_shift_capture: configpkg.MouseShiftCapture,
     macos_non_native_fullscreen: configpkg.NonNativeFullscreen,
-    macos_option_as_alt: ?configpkg.OptionAsAlt,
+    macos_option_as_alt: ?input.OptionAsAlt,
     selection_clear_on_copy: bool,
     selection_clear_on_typing: bool,
     vt_kam_allowed: bool,
@@ -1130,7 +1130,7 @@ fn childExited(self: *Surface, info: apprt.surface.Message.ChildExited) void {
         // so that we can close the terminal. We close the terminal on
         // any key press that encodes a character.
         t.modes.set(.disable_keyboard, false);
-        t.screen.kitty_keyboard.set(.set, .{});
+        t.screen.kitty_keyboard.set(.set, .disabled);
     }
 
     // Waiting after command we stop here. The terminal is updated, our
@@ -2611,56 +2611,32 @@ fn encodeKey(
     event: input.KeyEvent,
     insp_ev: ?*inspectorpkg.key.Event,
 ) !?termio.Message.WriteReq {
-    // Build up our encoder. Under different modes and
-    // inputs there are many keybindings that result in no encoding
-    // whatsoever.
-    const enc: input.KeyEncoder = enc: {
-        const option_as_alt: configpkg.OptionAsAlt = self.config.macos_option_as_alt orelse detect: {
-            // Non-macOS doesn't use this value so ignore.
-            if (comptime builtin.os.tag != .macos) break :detect .false;
-
-            // If we don't have alt pressed, it doesn't matter what this
-            // config is so we can just say "false" and break out and avoid
-            // more expensive checks below.
-            if (!event.mods.alt) break :detect .false;
-
-            // Alt is pressed, we're on macOS. We break some encapsulation
-            // here and assume libghostty for ease...
-            break :detect self.rt_app.keyboardLayout().detectOptionAsAlt();
-        };
-
-        self.renderer_state.mutex.lock();
-        defer self.renderer_state.mutex.unlock();
-        const t = &self.io.terminal;
-        break :enc .{
-            .event = event,
-            .macos_option_as_alt = option_as_alt,
-            .alt_esc_prefix = t.modes.get(.alt_esc_prefix),
-            .cursor_key_application = t.modes.get(.cursor_keys),
-            .keypad_key_application = t.modes.get(.keypad_keys),
-            .ignore_keypad_with_numlock = t.modes.get(.ignore_keypad_with_numlock),
-            .modify_other_keys_state_2 = t.flags.modify_other_keys_2,
-            .kitty_flags = t.screen.kitty_keyboard.current(),
-        };
-    };
-
     const write_req: termio.Message.WriteReq = req: {
+        // Build our encoding options, which requires the lock.
+        const encoding_opts = self.encodeKeyOpts();
+
         // Try to write the input into a small array. This fits almost
         // every scenario. Larger situations can happen due to long
         // pre-edits.
         var data: termio.Message.WriteReq.Small.Array = undefined;
-        if (enc.encode(&data)) |seq| {
+        var writer: std.Io.Writer = .fixed(&data);
+        if (input.key_encode.encode(
+            &writer,
+            event,
+            encoding_opts,
+        )) {
+            const written = writer.buffered();
+
             // Special-case: we did nothing.
-            if (seq.len == 0) return null;
+            if (written.len == 0) return null;
 
             break :req .{ .small = .{
                 .data = data,
-                .len = @intCast(seq.len),
+                .len = @intCast(written.len),
             } };
         } else |err| switch (err) {
             // Means we need to allocate
-            error.OutOfMemory => {},
-            else => return err,
+            error.WriteFailed => {},
         }
 
         // We need to allocate. We allocate double the UTF-8 length
@@ -2669,16 +2645,23 @@ fn encodeKey(
         // typing this where we don't have enough space is a long preedit,
         // and in that case the size we need is exactly the UTF-8 length,
         // so the double is being safe.
-        const buf = try self.alloc.alloc(u8, @max(
-            event.utf8.len * 2,
-            data.len * 2,
-        ));
-        defer self.alloc.free(buf);
+        var alloc_writer: std.Io.Writer.Allocating = try .initCapacity(
+            self.alloc,
+            @max(event.utf8.len * 2, data.len * 2),
+        );
+        defer alloc_writer.deinit();
 
         // This results in a double allocation but this is such an unlikely
         // path the performance impact is unimportant.
-        const seq = try enc.encode(buf);
-        break :req try termio.Message.WriteReq.init(self.alloc, seq);
+        try input.key_encode.encode(
+            &alloc_writer.writer,
+            event,
+            encoding_opts,
+        );
+        break :req try termio.Message.WriteReq.init(
+            self.alloc,
+            alloc_writer.writer.buffered(),
+        );
     };
 
     // Copy the encoded data into the inspector event if we have one.
@@ -2698,6 +2681,28 @@ fn encodeKey(
     return write_req;
 }
 
+fn encodeKeyOpts(self: *const Surface) input.key_encode.Options {
+    self.renderer_state.mutex.lock();
+    defer self.renderer_state.mutex.unlock();
+    const t = &self.io.terminal;
+
+    var opts: input.key_encode.Options = .fromTerminal(t);
+    if (comptime builtin.os.tag != .macos) return opts;
+
+    opts.macos_option_as_alt = self.config.macos_option_as_alt orelse detect: {
+        // If we don't have alt pressed, it doesn't matter what this
+        // config is so we can just say "false" and break out and avoid
+        // more expensive checks below.
+        if (!self.mouse.mods.alt) break :detect .false;
+
+        // Alt is pressed, we're on macOS. We break some encapsulation
+        // here and assume libghostty for ease...
+        break :detect self.rt_app.keyboardLayout().detectOptionAsAlt();
+    };
+
+    return opts;
+}
+
 /// Sends text as-is to the terminal without triggering any keyboard
 /// protocol. This will treat the input text as if it was pasted
 /// from the clipboard so the same logic will be applied. Namely,

src/config.zig

@@ -29,7 +29,6 @@ pub const Keybinds = Config.Keybinds;
 pub const MouseShiftCapture = Config.MouseShiftCapture;
 pub const MouseScrollMultiplier = Config.MouseScrollMultiplier;
 pub const NonNativeFullscreen = Config.NonNativeFullscreen;
-pub const OptionAsAlt = Config.OptionAsAlt;
 pub const RepeatableCodepointMap = Config.RepeatableCodepointMap;
 pub const RepeatableFontVariation = Config.RepeatableFontVariation;
 pub const RepeatableString = Config.RepeatableString;

src/config/Config.zig

@@ -2861,7 +2861,7 @@ keybind: Keybinds = .{},
 ///
 /// The values `left` or `right` enable this for the left or right *Option*
 /// key, respectively.
-@"macos-option-as-alt": ?OptionAsAlt = null,
+@"macos-option-as-alt": ?inputpkg.OptionAsAlt = null,
 
 /// Whether to enable the macOS window shadow. The default value is true.
 /// With some window managers and window transparency settings, you may
@@ -4821,14 +4821,6 @@ pub const NonNativeFullscreen = enum(c_int) {
     @"padded-notch",
 };
 
-/// Valid values for macos-option-as-alt.
-pub const OptionAsAlt = enum {
-    false,
-    true,
-    left,
-    right,
-};
-
 pub const WindowPaddingColor = enum {
     background,
     extend,

src/input.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const builtin = @import("builtin");
 
+const config = @import("input/config.zig");
 const mouse = @import("input/mouse.zig");
 const key = @import("input/key.zig");
 const keyboard = @import("input/keyboard.zig");
@@ -8,6 +9,7 @@ const keyboard = @import("input/keyboard.zig");
 pub const command = @import("input/command.zig");
 pub const function_keys = @import("input/function_keys.zig");
 pub const keycodes = @import("input/keycodes.zig");
+pub const key_encode = @import("input/key_encode.zig");
 pub const kitty = @import("input/kitty.zig");
 pub const paste = @import("input/paste.zig");
 
@@ -18,13 +20,13 @@ pub const Command = command.Command;
 pub const Link = @import("input/Link.zig");
 pub const Key = key.Key;
 pub const KeyboardLayout = keyboard.Layout;
-pub const KeyEncoder = @import("input/KeyEncoder.zig");
 pub const KeyEvent = key.KeyEvent;
 pub const InspectorMode = Binding.Action.InspectorMode;
 pub const Mods = key.Mods;
 pub const MouseButton = mouse.Button;
 pub const MouseButtonState = mouse.ButtonState;
 pub const MousePressureStage = mouse.PressureStage;
+pub const OptionAsAlt = config.OptionAsAlt;
 pub const ScrollMods = mouse.ScrollMods;
 pub const SplitFocusDirection = Binding.Action.SplitFocusDirection;
 pub const SplitResizeDirection = Binding.Action.SplitResizeDirection;

src/input/config.zig

@@ -0,0 +1,8 @@
+/// Determines the macOS option key behavior. See the config
+/// `macos-option-as-alt` for a lot more details.
+pub const OptionAsAlt = enum(c_int) {
+    false,
+    true,
+    left,
+    right,
+};

src/input/function_keys.zig

@@ -293,6 +293,11 @@ fn pcStyle(comptime fmt: []const u8) []Entry {
 
 test "keys" {
     const testing = std.testing;
+    switch (@import("terminal_options").artifact) {
+        .ghostty => {},
+        // Don't want to bring in termio into libghostty-vt
+        .lib => return error.SkipZigTest,
+    }
 
     // Force resolution for comptime evaluation.
     _ = keys;

src/input/key.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 const builtin = @import("builtin");
 const Allocator = std.mem.Allocator;
 const cimgui = @import("cimgui");
-const config = @import("../config.zig");
+const OptionAsAlt = @import("config.zig").OptionAsAlt;
 
 /// A generic key input event. This is the information that is necessary
 /// regardless of apprt in order to generate the proper terminal
@@ -146,7 +146,7 @@ pub const Mods = packed struct(Mods.Backing) {
     /// Return the mods to use for key translation. This handles settings
     /// like macos-option-as-alt. The translation mods should be used for
     /// translation but never sent back in for the key callback.
-    pub fn translation(self: Mods, option_as_alt: config.OptionAsAlt) Mods {
+    pub fn translation(self: Mods, option_as_alt: OptionAsAlt) Mods {
         var result = self;
 
         // macos-option-as-alt for darwin

src/input/keyboard.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const OptionAsAlt = @import("../config.zig").OptionAsAlt;
+const OptionAsAlt = @import("config.zig").OptionAsAlt;
 
 /// Keyboard layouts.
 ///

src/lib_vt.zig

@@ -72,10 +72,22 @@ pub const input = struct {
     // the input package because the full package brings in too many
     // other dependencies.
     const paste = @import("input/paste.zig");
+    const key = @import("input/key.zig");
+    const key_encode = @import("input/key_encode.zig");
+
+    // Paste-related APIs
     pub const PasteError = paste.Error;
     pub const PasteOptions = paste.Options;
     pub const isSafePaste = paste.isSafe;
     pub const encodePaste = paste.encode;
+
+    // Key encoding
+    pub const Key = key.Key;
+    pub const KeyAction = key.Action;
+    pub const KeyEvent = key.KeyEvent;
+    pub const KeyMods = key.Mods;
+    pub const KeyEncodeOptions = key_encode.Options;
+    pub const encodeKey = key_encode.encode;
 };
 
 comptime {

src/terminal/kitty/key.zig

@@ -8,7 +8,7 @@ const std = @import("std");
 pub const FlagStack = struct {
     const len = 8;
 
-    flags: [len]Flags = @splat(.{}),
+    flags: [len]Flags = @splat(.disabled),
     idx: u3 = 0,
 
     /// Return the current stack value
@@ -51,12 +51,12 @@ pub const FlagStack = struct {
         // could send a huge number of pop commands to waste cpu.
         if (n >= self.flags.len) {
             self.idx = 0;
-            self.flags = @splat(.{});
+            self.flags = @splat(.disabled);
             return;
         }
 
         for (0..n) |_| {
-            self.flags[self.idx] = .{};
+            self.flags[self.idx] = .disabled;
             self.idx -%= 1;
         }
     }
@@ -83,6 +83,15 @@ pub const Flags = packed struct(u5) {
     report_all: bool = false,
     report_associated: bool = false,
 
+    /// Kitty keyboard protocol disabled (all flags off).
+    pub const disabled: Flags = .{
+        .disambiguate = false,
+        .report_events = false,
+        .report_alternates = false,
+        .report_all = false,
+        .report_associated = false,
+    };
+
     /// Sets all modes on.
     pub const @"true": Flags = .{
         .disambiguate = true,