feat: allow handle virtual keyboard input by WM

anvil/src/input_handler.rs

@@ -7,6 +7,12 @@ use crate::udev::UdevData;
 #[cfg(feature = "udev")]
 use smithay::backend::renderer::DebugFlags;
 
+#[cfg(any(feature = "winit", feature = "x11", feature = "udev"))]
+use smithay::backend::input::AbsolutePositionEvent;
+use smithay::input::keyboard::KeyboardHandle;
+use smithay::input::keyboard::Keycode;
+use smithay::utils::SERIAL_COUNTER;
+use smithay::wayland::virtual_keyboard::VirtualKeyboardHandler;
 use smithay::{
     backend::input::{
         self, Axis, AxisSource, Event, InputBackend, InputEvent, KeyState, KeyboardKeyEvent,
@@ -30,9 +36,6 @@ use smithay::{
     },
 };
 
-#[cfg(any(feature = "winit", feature = "x11", feature = "udev"))]
-use smithay::backend::input::AbsolutePositionEvent;
-
 #[cfg(any(feature = "winit", feature = "x11"))]
 use smithay::output::Output;
 use tracing::{debug, error, info};
@@ -1327,3 +1330,26 @@ fn process_keyboard_shortcut(modifiers: ModifiersState, keysym: Keysym) -> Optio
         None
     }
 }
+
+impl<BackendData: Backend> VirtualKeyboardHandler for AnvilState<BackendData> {
+    fn on_keyboard_event(
+        &mut self,
+        keycode: Keycode,
+        state: KeyState,
+        time: u32,
+        keyboard: KeyboardHandle<Self>,
+    ) {
+        let serial = SERIAL_COUNTER.next_serial();
+        keyboard.input(self, keycode, state, serial, time, |_, _, _| {
+            FilterResult::Forward::<bool>
+        });
+    }
+    pub fn on_keyboard_modifiers(
+        &mut self,
+        depressed_mods: ModMask,
+        latched_mods: ModMask,
+        locked_mods: ModMask,
+        keyboard: KeyboardHandle<Self>,
+    ) {
+    }
+}

src/input/keyboard/keymap_file.rs

@@ -85,6 +85,12 @@ impl KeymapFile {
         Ok(())
     }
 
+    #[cfg(feature = "wayland_frontend")]
+    /// Send keymap string for processing
+    pub fn keymap_string(&self) -> String {
+        self.keymap.clone()
+    }
+
     /// Send the keymap contained within to a WlKeyboard
     pub fn send(
         &self,

src/input/keyboard/mod.rs

@@ -710,11 +710,11 @@ impl<D: SeatHandler + 'static> KeyboardHandle<D> {
         focus: &Option<&mut <D as SeatHandler>::KeyboardFocus>,
         keymap: &xkb::Keymap,
         mods: ModifiersState,
-    ) {
+    ) -> bool {
         let mut keymap_file = self.arc.keymap.lock().unwrap();
         keymap_file.change_keymap(keymap);
 
-        self.send_keymap(data, focus, &keymap_file, mods);
+        return self.send_keymap(data, focus, &keymap_file, mods);
     }
 
     /// Send a new wl_keyboard keymap, without updating the internal keymap.

src/wayland/virtual_keyboard/mod.rs

@@ -13,7 +13,11 @@
 //! # use smithay::wayland::compositor::{CompositorHandler, CompositorState, CompositorClientState};
 //! use smithay::wayland::virtual_keyboard::VirtualKeyboardManagerState;
 //! use smithay::reexports::wayland_server::{Display, protocol::wl_surface::WlSurface};
+//! use smithay::input::keyboard::KeyboardHandle;
+//! use smithay::backend::input::{KeyState, Keycode};
+//! use smithay::wayland::virtual_keyboard::VirtualKeyboardHandler;
 //! # use smithay::reexports::wayland_server::Client;
+//! use xkbcommon::xkb::ModMask;
 //!
 //! # struct State { seat_state: SeatState<Self> };
 //!
@@ -47,6 +51,22 @@
 //! #     fn client_compositor_state<'a>(&self, client: &'a Client) -> &'a CompositorClientState { unimplemented!() }
 //! #     fn commit(&mut self, surface: &WlSurface) {}
 //! # }
+//! impl VirtualKeyboardHandler for State {
+//!    fn on_keyboard_event(
+//!     &mut self,
+//!     keycode: Keycode,
+//!     state: KeyState,
+//!     time: u32,
+//!     keyboard: KeyboardHandle<Self>,
+//!  ) { }
+//!    fn on_keyboard_modifiers(
+//!        &mut self,
+//!        depressed_mods: ModMask,
+//!        latched_mods: ModMask,
+//!        locked_mods: ModMask,
+//!        keyboard: KeyboardHandle<Self>,
+//!    ) { }
+//! }
 //! # delegate_compositor!(State);
 //! ```
 //!
@@ -57,9 +77,11 @@ use wayland_protocols_misc::zwp_virtual_keyboard_v1::server::{
 };
 use wayland_server::{backend::GlobalId, Client, DataInit, Dispatch, DisplayHandle, GlobalDispatch, New};
 
-use crate::input::{Seat, SeatHandler};
-
 use self::virtual_keyboard_handle::VirtualKeyboardHandle;
+use crate::backend::input::{KeyState, Keycode};
+use crate::input::keyboard::KeyboardHandle;
+use crate::input::{Seat, SeatHandler};
+use xkbcommon::xkb::ModMask;
 
 const MANAGER_VERSION: u32 = 1;
 
@@ -91,6 +113,27 @@ where
     display.create_global::<D, ZwpVirtualKeyboardManagerV1, _>(MANAGER_VERSION, data)
 }
 
+/// Handle keyboard event
+pub trait VirtualKeyboardHandler: SeatHandler {
+    /// Handle KeyboardEvent as usual
+    fn on_keyboard_event(
+        &mut self,
+        keycode: Keycode,
+        state: KeyState,
+        time: u32,
+        keyboard: KeyboardHandle<Self>,
+    );
+
+    /// Handle modifiers event as usual
+    fn on_keyboard_modifiers(
+        &mut self,
+        depressed_mods: ModMask,
+        latched_mods: ModMask,
+        locked_mods: ModMask,
+        keyboard: KeyboardHandle<Self>,
+    );
+}
+
 impl VirtualKeyboardManagerState {
     /// Initialize a virtual keyboard manager global.
     pub fn new<D, F>(display: &DisplayHandle, filter: F) -> Self

src/wayland/virtual_keyboard/virtual_keyboard_handle.rs

@@ -10,36 +10,41 @@ use wayland_protocols_misc::zwp_virtual_keyboard_v1::server::zwp_virtual_keyboar
     self, ZwpVirtualKeyboardV1,
 };
 use wayland_server::{
-    backend::ClientId,
-    protocol::wl_keyboard::{KeyState, KeymapFormat},
-    Client, DataInit, Dispatch, DisplayHandle, Resource,
+    backend::ClientId, protocol::wl_keyboard::KeymapFormat, Client, DataInit, Dispatch, DisplayHandle,
+    Resource,
 };
 use xkbcommon::xkb;
 
-use crate::input::keyboard::{KeyboardTarget, KeymapFile, ModifiersState};
+use crate::input::keyboard::{KeyboardTarget, ModifiersState};
+use crate::wayland::input_method::InputMethodSeat;
 use crate::{
     input::{Seat, SeatHandler},
     utils::SERIAL_COUNTER,
     wayland::seat::{keyboard::for_each_focused_kbds, WaylandFocus},
 };
 
-use super::VirtualKeyboardManagerState;
+use crate::backend::input::KeyState;
+
+use super::{VirtualKeyboardHandler, VirtualKeyboardManagerState};
 
 #[derive(Debug, Default)]
 pub(crate) struct VirtualKeyboard {
     state: Option<VirtualKeyboardState>,
 }
 
 struct VirtualKeyboardState {
-    keymap: KeymapFile,
+    keymap: xkb::Keymap,
     mods: ModifiersState,
     state: xkb::State,
 }
 
 impl fmt::Debug for VirtualKeyboardState {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("VirtualKeyboardState")
-            .field("keymap", &self.keymap)
+            .field(
+                "keymap",
+                &self.keymap.get_as_string(xkb::KEYMAP_FORMAT_USE_ORIGINAL),
+            )
             .field("mods", &self.mods)
             .field("state", &self.state.get_raw_ptr())
             .finish()
@@ -75,21 +80,35 @@ impl<D> Dispatch<ZwpVirtualKeyboardV1, VirtualKeyboardUserData<D>, D> for Virtua
 where
     D: Dispatch<ZwpVirtualKeyboardV1, VirtualKeyboardUserData<D>>,
     D: SeatHandler + 'static,
+    D: VirtualKeyboardHandler,
     <D as SeatHandler>::KeyboardFocus: WaylandFocus,
 {
     fn request(
         user_data: &mut D,
-        _client: &Client,
+        client: &Client,
         virtual_keyboard: &ZwpVirtualKeyboardV1,
         request: zwp_virtual_keyboard_v1::Request,
         data: &VirtualKeyboardUserData<D>,
         _dh: &DisplayHandle,
         _data_init: &mut DataInit<'_, D>,
     ) {
+        let ime_keyboard_grabbed = data
+            .seat
+            .input_method()
+            .inner
+            .lock()
+            .unwrap()
+            .keyboard_grab
+            .inner
+            .lock()
+            .unwrap()
+            .grab
+            .clone();
         match request {
             zwp_virtual_keyboard_v1::Request::Keymap { format, fd, size } => {
                 update_keymap(data, format, fd, size as usize);
             }
+
             zwp_virtual_keyboard_v1::Request::Key { time, key, state } => {
                 // Ensure keymap was initialized.
                 let mut virtual_data = data.handle.inner.lock().unwrap();
@@ -100,25 +119,41 @@ where
                         return;
                     }
                 };
-
                 // Ensure virtual keyboard's keymap is active.
                 let keyboard_handle = data.seat.get_keyboard().unwrap();
-                let mut internal = keyboard_handle.arc.internal.lock().unwrap();
-                let focus = internal.focus.as_mut().map(|(focus, _)| focus);
-                keyboard_handle.send_keymap(user_data, &focus, &vk_state.keymap, vk_state.mods);
-
-                if let Some(wl_surface) = focus.and_then(|f| f.wl_surface()) {
-                    for_each_focused_kbds(&data.seat, &wl_surface, |kbd| {
-                        // This should be wl_keyboard::KeyState, but the protocol does not state
-                        // the parameter is an enum.
-                        let key_state = if state == 1 {
-                            KeyState::Pressed
-                        } else {
-                            KeyState::Released
-                        };
-
-                        kbd.key(SERIAL_COUNTER.next_serial().0, time, key, key_state);
-                    });
+
+                if ime_keyboard_grabbed.is_some()
+                    && ime_keyboard_grabbed.unwrap().client().unwrap() == *client
+                {
+                    use wayland_server::protocol::wl_keyboard::KeyState;
+                    let mut internal = keyboard_handle.arc.internal.lock().unwrap();
+                    let focus = internal.focus.as_mut().map(|(focus, _)| focus);
+                    keyboard_handle.change_keymap(user_data, &focus, &vk_state.keymap, vk_state.mods);
+                    if let Some(wl_surface) = focus.and_then(|f| f.wl_surface()) {
+                        for_each_focused_kbds(&data.seat, &wl_surface, |kbd| {
+                            // This should be wl_keyboard::KeyState, but the protocol does not state
+                            // the parameter is an enum.
+                            let key_state = if state == 1 {
+                                KeyState::Pressed
+                            } else {
+                                KeyState::Released
+                            };
+
+                            kbd.key(SERIAL_COUNTER.next_serial().0, time, key, key_state);
+                        });
+                    }
+                } else {
+                    {
+                        let mut internal = keyboard_handle.arc.internal.lock().unwrap();
+                        let focus = internal.focus.as_mut().map(|(focus, _)| focus);
+                        keyboard_handle.change_keymap(user_data, &focus, &vk_state.keymap, vk_state.mods);
+                    }
+                    let key_state = if state == 1 {
+                        KeyState::Pressed
+                    } else {
+                        KeyState::Released
+                    };
+                    user_data.on_keyboard_event((key + 8).into(), key_state, time, keyboard_handle);
                 }
             }
             zwp_virtual_keyboard_v1::Request::Modifiers {
@@ -145,17 +180,27 @@ where
 
                 // Ensure virtual keyboard's keymap is active.
                 let keyboard_handle = data.seat.get_keyboard().unwrap();
-                let mut internal = keyboard_handle.arc.internal.lock().unwrap();
-                let focus = internal.focus.as_mut().map(|(focus, _)| focus);
-                let keymap_changed =
-                    keyboard_handle.send_keymap(user_data, &focus, &state.keymap, state.mods);
-
-                // Report modifiers change to all keyboards.
-                if !keymap_changed {
-                    if let Some(focus) = focus {
-                        focus.modifiers(&data.seat, user_data, state.mods, SERIAL_COUNTER.next_serial());
+                {
+                    let mut internal = keyboard_handle.arc.internal.lock().unwrap();
+                    let focus = internal.focus.as_mut().map(|(focus, _)| focus);
+                    let keymap_changed =
+                        keyboard_handle.change_keymap(user_data, &focus, &state.keymap, state.mods);
+                    if !keymap_changed {
+                        if let Some(focus) = focus {
+                            focus.modifiers(&data.seat, user_data, state.mods, SERIAL_COUNTER.next_serial());
+                        }
                     }
                 }
+                if !ime_keyboard_grabbed.is_some()
+                    || !(ime_keyboard_grabbed.unwrap().client().unwrap() == *client)
+                {
+                    user_data.on_keyboard_modifiers(
+                        mods_depressed,
+                        mods_latched,
+                        mods_locked,
+                        keyboard_handle,
+                    );
+                }
             }
             zwp_virtual_keyboard_v1::Request::Destroy => {
                 // Nothing to do
@@ -213,7 +258,7 @@ where
     let mods = inner.state.take().map(|state| state.mods).unwrap_or_default();
     inner.state = Some(VirtualKeyboardState {
         mods,
-        keymap: KeymapFile::new(&new_keymap),
+        keymap: new_keymap.clone(),
         state: xkb::State::new(&new_keymap),
     });
 }