From 56b97c01fefbef6ce7599bc3857eed51e8bfa946 Mon Sep 17 00:00:00 2001 From: Arnold Loubriat Date: Sun, 23 Nov 2025 17:37:03 +0100 Subject: [PATCH 1/7] Proof-of-concept embedded screen reader --- Cargo.lock | 9 + Cargo.toml | 1 + masonry_screen_reader/Cargo.toml | 17 ++ masonry_screen_reader/src/adapter.rs | 269 +++++++++++++++++++++++++ masonry_screen_reader/src/filter.rs | 1 + masonry_screen_reader/src/lib.rs | 11 + masonry_winit/Cargo.toml | 5 +- masonry_winit/src/event_loop_runner.rs | 36 ++++ 8 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 masonry_screen_reader/Cargo.toml create mode 100644 masonry_screen_reader/src/adapter.rs create mode 100644 masonry_screen_reader/src/filter.rs create mode 100644 masonry_screen_reader/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index eea55d345..0dc3a9640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2183,6 +2183,14 @@ dependencies = [ "web-time", ] +[[package]] +name = "masonry_screen_reader" +version = "0.1.0" +dependencies = [ + "accesskit", + "accesskit_consumer", +] + [[package]] name = "masonry_testing" version = "0.4.0" @@ -2206,6 +2214,7 @@ dependencies = [ "copypasta", "masonry", "masonry_core", + "masonry_screen_reader", "pollster", "tracing", "tracing-tracy", diff --git a/Cargo.toml b/Cargo.toml index 50fb07ee8..08a7d5e09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "xilem_core", "masonry", "masonry_core", + "masonry_screen_reader", "masonry_testing", "masonry_winit", diff --git a/masonry_screen_reader/Cargo.toml b/masonry_screen_reader/Cargo.toml new file mode 100644 index 000000000..61962e415 --- /dev/null +++ b/masonry_screen_reader/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "masonry_screen_reader" +version = "0.1.0" +description = "Screen reader debug output for Masonry accessibility trees." +keywords = ["gui", "ui", "accessibility", "screen-reader"] +categories = ["gui", "accessibility", "development-tools::debugging"] +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +accesskit.workspace = true +accesskit_consumer.workspace = true + +[lints] +workspace = true diff --git a/masonry_screen_reader/src/adapter.rs b/masonry_screen_reader/src/adapter.rs new file mode 100644 index 000000000..e8c27ccff --- /dev/null +++ b/masonry_screen_reader/src/adapter.rs @@ -0,0 +1,269 @@ +use accesskit::{Point, Role, Toggled, TreeUpdate}; +use accesskit_consumer::{Node, Tree, TreeChangeHandler}; + +use crate::filter::filter; + +fn describe_role(node: &Node<'_>) -> Option { + let role_desc = match node.role() { + Role::Button => "button", + Role::CheckBox => "checkbox", + Role::TextInput => "text input", + Role::MultilineTextInput => "multiline text input", + Role::Document => "document", + Role::ProgressIndicator => "progress indicator", + Role::ScrollBar => "scrollbar", + Role::ScrollView => "scroll view", + Role::Splitter => "splitter", + Role::Image => "image", + _ => return None, + }; + + Some(role_desc.to_string()) +} + +fn describe_value(node: &Node<'_>) -> Option { + node.value() + .map(|value| { + if value.is_empty() { + "blank".to_string() + } else if node.role() == Role::PasswordInput { + format!("{} characters", value.len()) + } else { + value + } + }) + .or_else(|| { + node.numeric_value().map(|numeric_value| { + let min = node.min_numeric_value(); + let max = node.max_numeric_value(); + + match (min, max) { + (Some(min), Some(max)) if max > min => { + let percentage = ((numeric_value - min) / (max - min)) * 100.0; + format!("{:.1}%", percentage) + } + _ => numeric_value.to_string(), + } + }) + }) +} + +fn describe_state(node: &Node<'_>) -> String { + let mut states = Vec::new(); + + if node.is_disabled() { + states.push("disabled"); + } + + if node.is_read_only_supported() && node.is_read_only() { + states.push("readonly"); + } + + if let Some(toggled) = node.toggled() { + match toggled { + Toggled::True => states.push("checked"), + Toggled::False => states.push("unchecked"), + Toggled::Mixed => states.push("partially checked"), + } + } + + states.join(", ") +} + +fn describe_node(node: &Node<'_>) -> String { + let mut parts = Vec::new(); + + if !node.label_comes_from_value() { + if let Some(label) = node.label() { + parts.push(label); + } + } else if let Some(value) = node.value() { + parts.push(value); + } + + if let Some(role_desc) = describe_role(node) { + parts.push(role_desc); + } + + let state_info = describe_state(node); + if !state_info.is_empty() { + parts.push(state_info); + } + + if let Some(value_info) = describe_value(node) { + parts.push(value_info); + } + + if let Some(placeholder) = node.placeholder() { + parts.push(format!("placeholder: {}", placeholder)); + } + + parts.join(", ") +} + +struct ScreenReaderChangeHandler { + messages: Vec, +} + +impl ScreenReaderChangeHandler { + fn new() -> Self { + Self { + messages: Vec::new(), + } + } +} + +impl TreeChangeHandler for ScreenReaderChangeHandler { + fn node_added(&mut self, _node: &Node<'_>) {} + + fn node_updated(&mut self, old_node: &Node<'_>, new_node: &Node<'_>) { + if new_node.is_focused() { + let old_toggled = old_node.toggled(); + let new_toggled = new_node.toggled(); + + if old_toggled != new_toggled { + let description = describe_node(new_node); + self.messages.push(format!("Updated: {}", description)); + } + } else if new_node.role() == Role::ProgressIndicator { + let old_value = old_node.numeric_value(); + let new_value = new_node.numeric_value(); + + if old_value != new_value + && new_value.is_some() + && let Some(value_desc) = describe_value(new_node) + { + self.messages.push(value_desc); + } + } + } + + fn focus_moved(&mut self, _old_node: Option<&Node<'_>>, new_node: Option<&Node<'_>>) { + if let Some(new_node) = new_node { + self.messages.push(describe_node(new_node)); + } + } + + fn node_removed(&mut self, _node: &Node<'_>) {} +} + +#[derive(Debug)] +enum State { + Inactive { is_host_focused: bool }, + Active { tree: Box }, +} + +impl Default for State { + fn default() -> Self { + Self::Inactive { + is_host_focused: false, + } + } +} + +/// A screen reader simulator that generates human-readable descriptions of accessibility tree changes. +/// +/// `ScreenReader` monitors accessibility tree updates and produces text descriptions of what +/// would be announced to screen reader users. It starts in an inactive state and becomes active +/// when the first tree update is received. +#[derive(Debug, Default)] +pub struct ScreenReader { + state: State, +} + +impl ScreenReader { + /// Creates a new `ScreenReader` in an inactive state. + /// + /// The screen reader will become active when the first accessibility tree update is received + /// via [`update`](Self::update). + pub fn new() -> Self { + Self { + state: State::Inactive { + is_host_focused: false, + }, + } + } + + /// Processes an accessibility tree update and returns descriptions of changes. + /// + /// On the first call, this activates the screen reader and initializes it with the provided + /// tree structure. Subsequent calls process tree changes and generate descriptions. + /// + /// # Returns + /// + /// A vector of strings describing the changes that would be announced to a screen reader user. + pub fn update(&mut self, update: TreeUpdate) -> Vec { + match &mut self.state { + State::Inactive { is_host_focused } => { + let tree = Box::new(Tree::new(update, *is_host_focused)); + + let messages = if let Some(focused_node) = tree.state().focus() { + vec![describe_node(&focused_node)] + } else { + Vec::new() + }; + + self.state = State::Active { tree }; + messages + } + State::Active { tree } => { + let mut change_handler = ScreenReaderChangeHandler::new(); + tree.update_and_process_changes(update, &mut change_handler); + change_handler.messages + } + } + } + + /// Updates the window focus state and returns any resulting announcements. + /// + /// This should be called when the application window gains or loses focus. + /// + /// # Arguments + /// + /// * `is_focused` - Whether the window has focus + /// + /// # Returns + /// + /// A vector of strings describing any changes that result from the focus state change. + pub fn update_window_focus_state(&mut self, is_focused: bool) -> Vec { + match &mut self.state { + State::Inactive { is_host_focused } => { + *is_host_focused = is_focused; + Vec::new() + } + State::Active { tree } => { + let mut change_handler = ScreenReaderChangeHandler::new(); + tree.update_host_focus_state_and_process_changes(is_focused, &mut change_handler); + change_handler.messages + } + } + } + + /// Performs a hit test at the given coordinates and returns a description of the element found. + /// + /// This simulates what a screen reader would announce when the user touches or hovers over + /// a specific point in the interface. + /// + /// # Arguments + /// + /// * `x` - The x coordinate in logical pixels + /// * `y` - The y coordinate in logical pixels + /// + /// # Returns + /// + /// A vector containing a description of the element at the given point. + pub fn hit_test(&self, x: f64, y: f64) -> Vec { + match &self.state { + State::Inactive { .. } => Vec::new(), + State::Active { tree } => { + let root = tree.state().root(); + let point = Point::new(x, y); + if let Some(node) = root.node_at_point(point, &filter) { + vec![describe_node(&node)] + } else { + Vec::new() + } + } + } + } +} diff --git a/masonry_screen_reader/src/filter.rs b/masonry_screen_reader/src/filter.rs new file mode 100644 index 000000000..b630ac65a --- /dev/null +++ b/masonry_screen_reader/src/filter.rs @@ -0,0 +1 @@ +pub(crate) use accesskit_consumer::common_filter as filter; diff --git a/masonry_screen_reader/src/lib.rs b/masonry_screen_reader/src/lib.rs new file mode 100644 index 000000000..3f5437d7c --- /dev/null +++ b/masonry_screen_reader/src/lib.rs @@ -0,0 +1,11 @@ +//! Screen reader debug output for Masonry accessibility trees. +//! +//! This crate provides debugging tools to simulate screen reader behavior by generating +//! human-readable descriptions of accessibility tree updates. It helps developers understand +//! what information would be announced to screen reader users as they interact with a Masonry +//! application. + +mod adapter; +mod filter; + +pub use adapter::ScreenReader; diff --git a/masonry_winit/Cargo.toml b/masonry_winit/Cargo.toml index 681079a9a..37e66fe1a 100644 --- a/masonry_winit/Cargo.toml +++ b/masonry_winit/Cargo.toml @@ -18,13 +18,16 @@ targets = [] cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"] [features] -default = [] +default = ["screen-reader"] # Enables tracing using tracy if the default Masonry tracing is used. # https://github.com/wolfpld/tracy can be connected to when this feature is enabled. tracy = ["dep:tracing-tracy", "dep:wgpu-profiler", "wgpu-profiler/tracy", "masonry_core/tracy"] +# Enables screen reader debug output +screen-reader = ["dep:masonry_screen_reader"] [dependencies] masonry_core.workspace = true +masonry_screen_reader = { path = "../masonry_screen_reader", optional = true } winit.workspace = true tracing = { workspace = true, features = ["default"] } tracing-tracy = { version = "0.11.4", optional = true } diff --git a/masonry_winit/src/event_loop_runner.rs b/masonry_winit/src/event_loop_runner.rs index b0c44f1da..9bce5c09e 100644 --- a/masonry_winit/src/event_loop_runner.rs +++ b/masonry_winit/src/event_loop_runner.rs @@ -112,6 +112,8 @@ pub struct Window { id: WindowId, pub(crate) handle: Arc, pub(crate) accesskit_adapter: Adapter, + #[cfg(feature = "screen-reader")] + pub(crate) screen_reader: masonry_screen_reader::ScreenReader, event_reducer: WindowEventReducer, pub(crate) render_root: RenderRoot, pub(crate) base_color: Color, @@ -133,6 +135,8 @@ impl Window { id: window_id, handle, accesskit_adapter, + #[cfg(feature = "screen-reader")] + screen_reader: masonry_screen_reader::ScreenReader::new(), event_reducer: WindowEventReducer::default(), render_root: RenderRoot::new( root_widget, @@ -456,6 +460,12 @@ impl MasonryState<'_> { scale_factor, ); + // Enable accessibility tree for screen reader adapter + #[cfg(feature = "screen-reader")] + window + .render_root + .handle_window_event(WindowEvent::EnableAccessTree); + tracing::debug!(window_id = window.id.trace(), handle=?handle_id, "creating window"); self.window_id_to_handle_id.insert(window.id, handle_id); self.windows.insert(handle_id, window); @@ -523,6 +533,13 @@ impl MasonryState<'_> { Self::render(surface, window, scene, &self.render_cx, &mut self.renderer); #[cfg(feature = "tracy")] drop(self.frame.take()); + #[cfg(feature = "screen-reader")] + { + let messages = window.screen_reader.update(tree_update.clone()); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } if let Some(tree_update) = tree_update { window.accesskit_adapter.update_if_active(|| tree_update); } @@ -685,6 +702,18 @@ impl MasonryState<'_> { } } WindowEventTranslation::Pointer(p) => { + #[cfg(feature = "screen-reader")] + { + use masonry_core::core::PointerEvent; + if let PointerEvent::Move(update) = &p { + let messages = window + .screen_reader + .hit_test(update.current.position.x, update.current.position.y); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } + } window.render_root.handle_pointer_event(p); } } @@ -715,6 +744,13 @@ impl MasonryState<'_> { window .render_root .handle_text_event(TextEvent::WindowFocusChange(new_focus)); + #[cfg(feature = "screen-reader")] + { + let messages = window.screen_reader.update_window_focus_state(new_focus); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } } _ => (), } From ec9ec2791ebff4c2e6a6b70aec90b5dfb48d8ad5 Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:14:29 +0100 Subject: [PATCH 2/7] Add copyright notices --- masonry_screen_reader/src/adapter.rs | 3 +++ masonry_screen_reader/src/filter.rs | 3 +++ masonry_screen_reader/src/lib.rs | 3 +++ 3 files changed, 9 insertions(+) diff --git a/masonry_screen_reader/src/adapter.rs b/masonry_screen_reader/src/adapter.rs index e8c27ccff..08461e060 100644 --- a/masonry_screen_reader/src/adapter.rs +++ b/masonry_screen_reader/src/adapter.rs @@ -1,3 +1,6 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + use accesskit::{Point, Role, Toggled, TreeUpdate}; use accesskit_consumer::{Node, Tree, TreeChangeHandler}; diff --git a/masonry_screen_reader/src/filter.rs b/masonry_screen_reader/src/filter.rs index b630ac65a..09f1ac0b3 100644 --- a/masonry_screen_reader/src/filter.rs +++ b/masonry_screen_reader/src/filter.rs @@ -1 +1,4 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + pub(crate) use accesskit_consumer::common_filter as filter; diff --git a/masonry_screen_reader/src/lib.rs b/masonry_screen_reader/src/lib.rs index 3f5437d7c..88d821aa4 100644 --- a/masonry_screen_reader/src/lib.rs +++ b/masonry_screen_reader/src/lib.rs @@ -1,3 +1,6 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + //! Screen reader debug output for Masonry accessibility trees. //! //! This crate provides debugging tools to simulate screen reader behavior by generating From 1d35ecc6e421d06d54d2bf1be4873c55593d8d02 Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:20:59 +0100 Subject: [PATCH 3/7] Fix build error and change some logic --- masonry_screen_reader/src/adapter.rs | 10 +++++----- masonry_winit/src/event_loop_runner.rs | 22 ++++++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/masonry_screen_reader/src/adapter.rs b/masonry_screen_reader/src/adapter.rs index 08461e060..056e10a82 100644 --- a/masonry_screen_reader/src/adapter.rs +++ b/masonry_screen_reader/src/adapter.rs @@ -254,17 +254,17 @@ impl ScreenReader { /// /// # Returns /// - /// A vector containing a description of the element at the given point. - pub fn hit_test(&self, x: f64, y: f64) -> Vec { + /// An optional string with a description of the element at the given point. + pub fn hit_test(&self, x: f64, y: f64) -> Option { match &self.state { - State::Inactive { .. } => Vec::new(), + State::Inactive { .. } => None, State::Active { tree } => { let root = tree.state().root(); let point = Point::new(x, y); if let Some(node) = root.node_at_point(point, &filter) { - vec![describe_node(&node)] + Some(describe_node(&node)) } else { - Vec::new() + None } } } diff --git a/masonry_winit/src/event_loop_runner.rs b/masonry_winit/src/event_loop_runner.rs index 9bce5c09e..975edb841 100644 --- a/masonry_winit/src/event_loop_runner.rs +++ b/masonry_winit/src/event_loop_runner.rs @@ -448,7 +448,7 @@ impl MasonryState<'_> { #[cfg(not(target_os = "ios"))] let size = handle.inner_size(); - let window = Window::new( + let mut window = Window::new( new_window.id, handle, adapter, @@ -533,14 +533,16 @@ impl MasonryState<'_> { Self::render(surface, window, scene, &self.render_cx, &mut self.renderer); #[cfg(feature = "tracy")] drop(self.frame.take()); - #[cfg(feature = "screen-reader")] - { - let messages = window.screen_reader.update(tree_update.clone()); - for msg in messages { - tracing::info!(target: "masonry_screen_reader", "{}", msg); - } - } + if let Some(tree_update) = tree_update { + #[cfg(feature = "screen-reader")] + { + let messages = window.screen_reader.update(tree_update.clone()); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } + window.accesskit_adapter.update_if_active(|| tree_update); } } @@ -706,10 +708,10 @@ impl MasonryState<'_> { { use masonry_core::core::PointerEvent; if let PointerEvent::Move(update) = &p { - let messages = window + let message = window .screen_reader .hit_test(update.current.position.x, update.current.position.y); - for msg in messages { + if let Some(msg) = message { tracing::info!(target: "masonry_screen_reader", "{}", msg); } } From 81d6c8e7052066524a938741535b83537832fcec Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:21:11 +0100 Subject: [PATCH 4/7] Rewrite doc --- masonry_screen_reader/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/masonry_screen_reader/src/lib.rs b/masonry_screen_reader/src/lib.rs index 88d821aa4..95a82ff6c 100644 --- a/masonry_screen_reader/src/lib.rs +++ b/masonry_screen_reader/src/lib.rs @@ -4,9 +4,14 @@ //! Screen reader debug output for Masonry accessibility trees. //! //! This crate provides debugging tools to simulate screen reader behavior by generating -//! human-readable descriptions of accessibility tree updates. It helps developers understand -//! what information would be announced to screen reader users as they interact with a Masonry -//! application. +//! human-readable descriptions of accessibility tree updates. +//! It helps developers understand what information would be announced to screen reader +//! users as they interact with a Masonry application. +//! +//! While this crate isn't trying to emulate a specific screen reader (yet), it aims to +//! emulate the *median* screen reader. +//! That means, among other things, that it shouldn't give users information that most +//! screen readers wouldn't give. mod adapter; mod filter; From 41504241488bdb1c3a339e424241814bc67386cd Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:26:44 +0100 Subject: [PATCH 5/7] Remove clippy lint --- masonry_screen_reader/src/adapter.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/masonry_screen_reader/src/adapter.rs b/masonry_screen_reader/src/adapter.rs index 056e10a82..978096d80 100644 --- a/masonry_screen_reader/src/adapter.rs +++ b/masonry_screen_reader/src/adapter.rs @@ -261,11 +261,8 @@ impl ScreenReader { State::Active { tree } => { let root = tree.state().root(); let point = Point::new(x, y); - if let Some(node) = root.node_at_point(point, &filter) { - Some(describe_node(&node)) - } else { - None - } + let node = root.node_at_point(point, &filter)?; + Some(describe_node(&node)) } } } From 05f6670eeb439f0e9fe0e632e8a236094acc34b9 Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:27:41 +0100 Subject: [PATCH 6/7] Add README --- masonry_screen_reader/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 masonry_screen_reader/README.md diff --git a/masonry_screen_reader/README.md b/masonry_screen_reader/README.md new file mode 100644 index 000000000..93cbf8033 --- /dev/null +++ b/masonry_screen_reader/README.md @@ -0,0 +1,13 @@ +# Masonry Screen reader + +Screen reader debug output for Masonry accessibility trees. + +This crate provides debugging tools to simulate screen reader behavior by generating +human-readable descriptions of accessibility tree updates. +It helps developers understand what information would be announced to screen reader +users as they interact with a Masonry application. + +While this crate isn't trying to emulate a specific screen reader (yet), it aims to +emulate the *median* screen reader. +That means, among other things, that it shouldn't give users information that most +screen readers wouldn't give. From 5c1e6612cb966bbfb74c70768dfab527a271ad4c Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Sun, 30 Nov 2025 18:33:41 +0100 Subject: [PATCH 7/7] Fix lint --- masonry_winit/src/event_loop_runner.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/masonry_winit/src/event_loop_runner.rs b/masonry_winit/src/event_loop_runner.rs index 975edb841..e49f3bcd2 100644 --- a/masonry_winit/src/event_loop_runner.rs +++ b/masonry_winit/src/event_loop_runner.rs @@ -448,6 +448,7 @@ impl MasonryState<'_> { #[cfg(not(target_os = "ios"))] let size = handle.inner_size(); + #[allow(unused_mut, reason = "Depends on the flags")] let mut window = Window::new( new_window.id, handle,