diff --git a/Cargo.lock b/Cargo.lock index eea55d345..0dc3a9640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2183,6 +2183,14 @@ dependencies = [ "web-time", ] +[[package]] +name = "masonry_screen_reader" +version = "0.1.0" +dependencies = [ + "accesskit", + "accesskit_consumer", +] + [[package]] name = "masonry_testing" version = "0.4.0" @@ -2206,6 +2214,7 @@ dependencies = [ "copypasta", "masonry", "masonry_core", + "masonry_screen_reader", "pollster", "tracing", "tracing-tracy", diff --git a/Cargo.toml b/Cargo.toml index 50fb07ee8..08a7d5e09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "xilem_core", "masonry", "masonry_core", + "masonry_screen_reader", "masonry_testing", "masonry_winit", diff --git a/masonry_screen_reader/Cargo.toml b/masonry_screen_reader/Cargo.toml new file mode 100644 index 000000000..61962e415 --- /dev/null +++ b/masonry_screen_reader/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "masonry_screen_reader" +version = "0.1.0" +description = "Screen reader debug output for Masonry accessibility trees." +keywords = ["gui", "ui", "accessibility", "screen-reader"] +categories = ["gui", "accessibility", "development-tools::debugging"] +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +accesskit.workspace = true +accesskit_consumer.workspace = true + +[lints] +workspace = true diff --git a/masonry_screen_reader/README.md b/masonry_screen_reader/README.md new file mode 100644 index 000000000..93cbf8033 --- /dev/null +++ b/masonry_screen_reader/README.md @@ -0,0 +1,13 @@ +# Masonry Screen reader + +Screen reader debug output for Masonry accessibility trees. + +This crate provides debugging tools to simulate screen reader behavior by generating +human-readable descriptions of accessibility tree updates. +It helps developers understand what information would be announced to screen reader +users as they interact with a Masonry application. + +While this crate isn't trying to emulate a specific screen reader (yet), it aims to +emulate the *median* screen reader. +That means, among other things, that it shouldn't give users information that most +screen readers wouldn't give. diff --git a/masonry_screen_reader/src/adapter.rs b/masonry_screen_reader/src/adapter.rs new file mode 100644 index 000000000..978096d80 --- /dev/null +++ b/masonry_screen_reader/src/adapter.rs @@ -0,0 +1,269 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + +use accesskit::{Point, Role, Toggled, TreeUpdate}; +use accesskit_consumer::{Node, Tree, TreeChangeHandler}; + +use crate::filter::filter; + +fn describe_role(node: &Node<'_>) -> Option { + let role_desc = match node.role() { + Role::Button => "button", + Role::CheckBox => "checkbox", + Role::TextInput => "text input", + Role::MultilineTextInput => "multiline text input", + Role::Document => "document", + Role::ProgressIndicator => "progress indicator", + Role::ScrollBar => "scrollbar", + Role::ScrollView => "scroll view", + Role::Splitter => "splitter", + Role::Image => "image", + _ => return None, + }; + + Some(role_desc.to_string()) +} + +fn describe_value(node: &Node<'_>) -> Option { + node.value() + .map(|value| { + if value.is_empty() { + "blank".to_string() + } else if node.role() == Role::PasswordInput { + format!("{} characters", value.len()) + } else { + value + } + }) + .or_else(|| { + node.numeric_value().map(|numeric_value| { + let min = node.min_numeric_value(); + let max = node.max_numeric_value(); + + match (min, max) { + (Some(min), Some(max)) if max > min => { + let percentage = ((numeric_value - min) / (max - min)) * 100.0; + format!("{:.1}%", percentage) + } + _ => numeric_value.to_string(), + } + }) + }) +} + +fn describe_state(node: &Node<'_>) -> String { + let mut states = Vec::new(); + + if node.is_disabled() { + states.push("disabled"); + } + + if node.is_read_only_supported() && node.is_read_only() { + states.push("readonly"); + } + + if let Some(toggled) = node.toggled() { + match toggled { + Toggled::True => states.push("checked"), + Toggled::False => states.push("unchecked"), + Toggled::Mixed => states.push("partially checked"), + } + } + + states.join(", ") +} + +fn describe_node(node: &Node<'_>) -> String { + let mut parts = Vec::new(); + + if !node.label_comes_from_value() { + if let Some(label) = node.label() { + parts.push(label); + } + } else if let Some(value) = node.value() { + parts.push(value); + } + + if let Some(role_desc) = describe_role(node) { + parts.push(role_desc); + } + + let state_info = describe_state(node); + if !state_info.is_empty() { + parts.push(state_info); + } + + if let Some(value_info) = describe_value(node) { + parts.push(value_info); + } + + if let Some(placeholder) = node.placeholder() { + parts.push(format!("placeholder: {}", placeholder)); + } + + parts.join(", ") +} + +struct ScreenReaderChangeHandler { + messages: Vec, +} + +impl ScreenReaderChangeHandler { + fn new() -> Self { + Self { + messages: Vec::new(), + } + } +} + +impl TreeChangeHandler for ScreenReaderChangeHandler { + fn node_added(&mut self, _node: &Node<'_>) {} + + fn node_updated(&mut self, old_node: &Node<'_>, new_node: &Node<'_>) { + if new_node.is_focused() { + let old_toggled = old_node.toggled(); + let new_toggled = new_node.toggled(); + + if old_toggled != new_toggled { + let description = describe_node(new_node); + self.messages.push(format!("Updated: {}", description)); + } + } else if new_node.role() == Role::ProgressIndicator { + let old_value = old_node.numeric_value(); + let new_value = new_node.numeric_value(); + + if old_value != new_value + && new_value.is_some() + && let Some(value_desc) = describe_value(new_node) + { + self.messages.push(value_desc); + } + } + } + + fn focus_moved(&mut self, _old_node: Option<&Node<'_>>, new_node: Option<&Node<'_>>) { + if let Some(new_node) = new_node { + self.messages.push(describe_node(new_node)); + } + } + + fn node_removed(&mut self, _node: &Node<'_>) {} +} + +#[derive(Debug)] +enum State { + Inactive { is_host_focused: bool }, + Active { tree: Box }, +} + +impl Default for State { + fn default() -> Self { + Self::Inactive { + is_host_focused: false, + } + } +} + +/// A screen reader simulator that generates human-readable descriptions of accessibility tree changes. +/// +/// `ScreenReader` monitors accessibility tree updates and produces text descriptions of what +/// would be announced to screen reader users. It starts in an inactive state and becomes active +/// when the first tree update is received. +#[derive(Debug, Default)] +pub struct ScreenReader { + state: State, +} + +impl ScreenReader { + /// Creates a new `ScreenReader` in an inactive state. + /// + /// The screen reader will become active when the first accessibility tree update is received + /// via [`update`](Self::update). + pub fn new() -> Self { + Self { + state: State::Inactive { + is_host_focused: false, + }, + } + } + + /// Processes an accessibility tree update and returns descriptions of changes. + /// + /// On the first call, this activates the screen reader and initializes it with the provided + /// tree structure. Subsequent calls process tree changes and generate descriptions. + /// + /// # Returns + /// + /// A vector of strings describing the changes that would be announced to a screen reader user. + pub fn update(&mut self, update: TreeUpdate) -> Vec { + match &mut self.state { + State::Inactive { is_host_focused } => { + let tree = Box::new(Tree::new(update, *is_host_focused)); + + let messages = if let Some(focused_node) = tree.state().focus() { + vec![describe_node(&focused_node)] + } else { + Vec::new() + }; + + self.state = State::Active { tree }; + messages + } + State::Active { tree } => { + let mut change_handler = ScreenReaderChangeHandler::new(); + tree.update_and_process_changes(update, &mut change_handler); + change_handler.messages + } + } + } + + /// Updates the window focus state and returns any resulting announcements. + /// + /// This should be called when the application window gains or loses focus. + /// + /// # Arguments + /// + /// * `is_focused` - Whether the window has focus + /// + /// # Returns + /// + /// A vector of strings describing any changes that result from the focus state change. + pub fn update_window_focus_state(&mut self, is_focused: bool) -> Vec { + match &mut self.state { + State::Inactive { is_host_focused } => { + *is_host_focused = is_focused; + Vec::new() + } + State::Active { tree } => { + let mut change_handler = ScreenReaderChangeHandler::new(); + tree.update_host_focus_state_and_process_changes(is_focused, &mut change_handler); + change_handler.messages + } + } + } + + /// Performs a hit test at the given coordinates and returns a description of the element found. + /// + /// This simulates what a screen reader would announce when the user touches or hovers over + /// a specific point in the interface. + /// + /// # Arguments + /// + /// * `x` - The x coordinate in logical pixels + /// * `y` - The y coordinate in logical pixels + /// + /// # Returns + /// + /// An optional string with a description of the element at the given point. + pub fn hit_test(&self, x: f64, y: f64) -> Option { + match &self.state { + State::Inactive { .. } => None, + State::Active { tree } => { + let root = tree.state().root(); + let point = Point::new(x, y); + let node = root.node_at_point(point, &filter)?; + Some(describe_node(&node)) + } + } + } +} diff --git a/masonry_screen_reader/src/filter.rs b/masonry_screen_reader/src/filter.rs new file mode 100644 index 000000000..09f1ac0b3 --- /dev/null +++ b/masonry_screen_reader/src/filter.rs @@ -0,0 +1,4 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) use accesskit_consumer::common_filter as filter; diff --git a/masonry_screen_reader/src/lib.rs b/masonry_screen_reader/src/lib.rs new file mode 100644 index 000000000..95a82ff6c --- /dev/null +++ b/masonry_screen_reader/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright 2025 the Xilem Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Screen reader debug output for Masonry accessibility trees. +//! +//! This crate provides debugging tools to simulate screen reader behavior by generating +//! human-readable descriptions of accessibility tree updates. +//! It helps developers understand what information would be announced to screen reader +//! users as they interact with a Masonry application. +//! +//! While this crate isn't trying to emulate a specific screen reader (yet), it aims to +//! emulate the *median* screen reader. +//! That means, among other things, that it shouldn't give users information that most +//! screen readers wouldn't give. + +mod adapter; +mod filter; + +pub use adapter::ScreenReader; diff --git a/masonry_winit/Cargo.toml b/masonry_winit/Cargo.toml index 681079a9a..37e66fe1a 100644 --- a/masonry_winit/Cargo.toml +++ b/masonry_winit/Cargo.toml @@ -18,13 +18,16 @@ targets = [] cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"] [features] -default = [] +default = ["screen-reader"] # Enables tracing using tracy if the default Masonry tracing is used. # https://github.com/wolfpld/tracy can be connected to when this feature is enabled. tracy = ["dep:tracing-tracy", "dep:wgpu-profiler", "wgpu-profiler/tracy", "masonry_core/tracy"] +# Enables screen reader debug output +screen-reader = ["dep:masonry_screen_reader"] [dependencies] masonry_core.workspace = true +masonry_screen_reader = { path = "../masonry_screen_reader", optional = true } winit.workspace = true tracing = { workspace = true, features = ["default"] } tracing-tracy = { version = "0.11.4", optional = true } diff --git a/masonry_winit/src/event_loop_runner.rs b/masonry_winit/src/event_loop_runner.rs index b0c44f1da..e49f3bcd2 100644 --- a/masonry_winit/src/event_loop_runner.rs +++ b/masonry_winit/src/event_loop_runner.rs @@ -112,6 +112,8 @@ pub struct Window { id: WindowId, pub(crate) handle: Arc, pub(crate) accesskit_adapter: Adapter, + #[cfg(feature = "screen-reader")] + pub(crate) screen_reader: masonry_screen_reader::ScreenReader, event_reducer: WindowEventReducer, pub(crate) render_root: RenderRoot, pub(crate) base_color: Color, @@ -133,6 +135,8 @@ impl Window { id: window_id, handle, accesskit_adapter, + #[cfg(feature = "screen-reader")] + screen_reader: masonry_screen_reader::ScreenReader::new(), event_reducer: WindowEventReducer::default(), render_root: RenderRoot::new( root_widget, @@ -444,7 +448,8 @@ impl MasonryState<'_> { #[cfg(not(target_os = "ios"))] let size = handle.inner_size(); - let window = Window::new( + #[allow(unused_mut, reason = "Depends on the flags")] + let mut window = Window::new( new_window.id, handle, adapter, @@ -456,6 +461,12 @@ impl MasonryState<'_> { scale_factor, ); + // Enable accessibility tree for screen reader adapter + #[cfg(feature = "screen-reader")] + window + .render_root + .handle_window_event(WindowEvent::EnableAccessTree); + tracing::debug!(window_id = window.id.trace(), handle=?handle_id, "creating window"); self.window_id_to_handle_id.insert(window.id, handle_id); self.windows.insert(handle_id, window); @@ -523,7 +534,16 @@ impl MasonryState<'_> { Self::render(surface, window, scene, &self.render_cx, &mut self.renderer); #[cfg(feature = "tracy")] drop(self.frame.take()); + if let Some(tree_update) = tree_update { + #[cfg(feature = "screen-reader")] + { + let messages = window.screen_reader.update(tree_update.clone()); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } + window.accesskit_adapter.update_if_active(|| tree_update); } } @@ -685,6 +705,18 @@ impl MasonryState<'_> { } } WindowEventTranslation::Pointer(p) => { + #[cfg(feature = "screen-reader")] + { + use masonry_core::core::PointerEvent; + if let PointerEvent::Move(update) = &p { + let message = window + .screen_reader + .hit_test(update.current.position.x, update.current.position.y); + if let Some(msg) = message { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } + } window.render_root.handle_pointer_event(p); } } @@ -715,6 +747,13 @@ impl MasonryState<'_> { window .render_root .handle_text_event(TextEvent::WindowFocusChange(new_focus)); + #[cfg(feature = "screen-reader")] + { + let messages = window.screen_reader.update_window_focus_state(new_focus); + for msg in messages { + tracing::info!(target: "masonry_screen_reader", "{}", msg); + } + } } _ => (), }