From 01110b5b2ed408bfe7b5497ebd14ac5e70bd8494 Mon Sep 17 00:00:00 2001 From: hangyu Date: Thu, 27 Aug 2026 17:24:05 -0700 Subject: [PATCH 1/7] tree UI --- .../accessibility_controller.dart | 258 ++++++++++++ .../accessibility/semantics_tree_pane.dart | 209 +++++++++- .../accessibility_controller_test.dart | 372 +++++++++++++++++- .../accessibility_screen_test.dart | 124 ++++++ 4 files changed, 953 insertions(+), 10 deletions(-) diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index 1fd71a307f6..74ff08a6159 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. +/// @docImport 'package:flutter/semantics.dart'; +library; + import 'dart:async'; import 'package:devtools_app_shared/service.dart'; @@ -12,6 +15,79 @@ import '../../service/service_extensions.dart' as extensions; import '../../shared/framework/screen.dart'; import '../../shared/framework/screen_controllers.dart'; import '../../shared/globals.dart'; +import '../../shared/primitives/trees.dart'; + +/// Represents a node in the semantics tree. +class SemanticsNodeModel extends TreeNode { + SemanticsNodeModel({ + required this.id, + this.label = '', + this.value = '', + this.hint = '', + this.tooltip = '', + this.increasedValue = '', + this.decreasedValue = '', + this.flags = const [], + this.actions = const [], + this.widgetName = '', + this.rectString = '', + this.transform, + }); + + /// The semantics node identifier, as provided by the Flutter framework. + final String id; + + /// The user-visible label announced by screen readers (maps to [SemanticsData.label]). + final String label; + + /// The current value of this node (e.g. the text in a text field) (maps to [SemanticsData.value]). + final String value; + + /// Additional hint text spoken after a delay (maps to [SemanticsData.hint]). + final String hint; + + /// A brief description of the widget the semantics node represents (maps to [SemanticsData.tooltip]). + final String tooltip; + + /// The value that the node will take if the user increases it (maps to [SemanticsData.increasedValue]). + final String increasedValue; + + /// The value that the node will take if the user decreases it (maps to [SemanticsData.decreasedValue]). + final String decreasedValue; + + /// Semantic flags active on this node (e.g. `'isButton'`, `'isHeader'`). + final List flags; + + /// Semantic actions that can be performed on this node (e.g. `'tap'`, `'scrollLeft'`). + final List actions; + + /// The name of the Flutter widget that produced this node, if available. + final String widgetName; + + /// Human-readable representation of this node's bounding rect (maps to [SemanticsData.rect]). + final String rectString; + + /// The transformation matrix to apply to this node's coordinate system (maps to [SemanticsData.transform]). + final List? transform; + + @override + SemanticsNodeModel shallowCopy() { + return SemanticsNodeModel( + id: id, + label: label, + value: value, + hint: hint, + tooltip: tooltip, + increasedValue: increasedValue, + decreasedValue: decreasedValue, + flags: flags, + actions: actions, + widgetName: widgetName, + rectString: rectString, + transform: transform, + ); + } +} /// Modes for brightness override in the accessibility controls. enum BrightnessOverride { @@ -40,6 +116,7 @@ class AccessibilityController extends DevToolsScreenController void init() { super.init(); _initServiceExtensionStates(); + _initSemanticsTree(); } void _initListeners() { @@ -50,6 +127,37 @@ class AccessibilityController extends DevToolsScreenController addAutoDisposeListener(highContrast, _onHighContrastChanged); } + void _initSemanticsTree() { + if (serviceConnection.serviceManager.isolateManager.mainIsolate.value != + null) { + unawaited(_autoLoadSemanticsTreeIfNeeded()); + } + addAutoDisposeListener( + serviceConnection.serviceManager.isolateManager.mainIsolate, + () { + if (serviceConnection.serviceManager.isolateManager.mainIsolate.value != + null) { + // Clear stale data from a previous isolate so the guard in + // _autoLoadSemanticsTreeIfNeeded doesn't skip the new load. + semanticsRoots.value = []; + semanticsTreeError.value = null; + unawaited(_autoLoadSemanticsTreeIfNeeded()); + } else { + semanticsRoots.value = []; + semanticsTreeError.value = null; + } + }, + ); + } + + Future _autoLoadSemanticsTreeIfNeeded() async { + if (semanticsRoots.value.isEmpty && + semanticsTreeError.value == null && + !semanticsTreeLoading.value) { + await loadSemanticsTree(); + } + } + void _initServiceExtensionStates() { final state = serviceConnection.serviceManager.serviceExtensionManager .getServiceExtensionState(extensions.brightnessMode.extension); @@ -109,13 +217,163 @@ class AccessibilityController extends DevToolsScreenController final screenReader = ValueNotifier(false); final highContrast = ValueNotifier(false); + final semanticsRoots = ValueNotifier>([]); + final semanticsTreeLoading = ValueNotifier(false); + final semanticsTreeError = ValueNotifier(null); + + Future loadSemanticsTree() async { + if (semanticsTreeLoading.value) return; + + final mainIsolate = + serviceConnection.serviceManager.isolateManager.mainIsolate.value; + if (mainIsolate == null) { + semanticsTreeError.value = + 'Failed to load semantics tree: no connected application.'; + return; + } + + semanticsTreeLoading.value = true; + semanticsTreeError.value = null; + // Intentionally do NOT clear semanticsRoots here so that the old tree + // remains visible while a refresh is in flight. + + try { + await serviceConnection.serviceManager.callServiceExtensionOnMainIsolate( + 'ext.flutter.accessibility.enableSemantics', + args: {'enabled': 'true'}, + ); + + final response = await serviceConnection.serviceManager + .callServiceExtensionOnMainIsolate( + 'ext.flutter.accessibility.getSemanticsTree', + ); + + final json = response.json; + if (json != null && json.containsKey('error')) { + throw Exception(json['error']); + } + + final rawData = json?['data']; + if (rawData == null) { + throw Exception( + 'Empty semantics tree returned from service extension.', + ); + } + + final roots = []; + if (rawData is Map) { + if (rawData.isNotEmpty) { + final rootId = rawData.containsKey('0') + ? '0' + : rawData.keys.first.toString(); + roots.add(_buildTreeFromNodesMap(rootId, rawData, {})); + } + } + + if (roots.isEmpty) { + throw Exception('No semantics nodes found in response.'); + } + + for (final root in roots) { + root.expandCascading(); + } + semanticsRoots.value = roots; + semanticsTreeError.value = null; + } catch (e, st) { + debugPrint('Error loading semantics tree: $e'); + debugPrint('$st'); + semanticsRoots.value = []; + semanticsTreeError.value = 'Failed to load semantics tree: $e'; + } finally { + semanticsTreeLoading.value = false; + } + } + + SemanticsNodeModel _buildTreeFromNodesMap( + String nodeId, + Map nodesMap, + Set visited, + ) { + if (!visited.add(nodeId)) { + return SemanticsNodeModel(id: nodeId); + } + + final json = + (nodesMap[nodeId] as Map?) ?? + {'id': nodeId}; + final node = _parseSemanticsNode(json); + + final childIds = + (json['childrenInTraversalOrder'] as List?) + ?.map((e) => e.toString()) + .toList() ?? + (json['childrenInHitTestOrder'] as List?) + ?.map((e) => e.toString()) + .toList() ?? + const []; + + for (final childId in childIds) { + if (nodesMap.containsKey(childId)) { + final childNode = _buildTreeFromNodesMap(childId, nodesMap, visited); + node.addChild(childNode); + } + } + + return node; + } + + SemanticsNodeModel _parseSemanticsNode(Map json) { + final rect = json['rect'] as Map?; + final rectString = rect != null + ? 'rect: Rect.fromLTWH(${rect['left']}, ${rect['top']}, ${rect['width']}, ${rect['height']})' + : 'Rect.zero'; + + final flags = (json['flags'] as List?)?.cast() ?? const []; + final actions = (json['actions'] as List?)?.cast() ?? const []; + final transform = (json['transform'] as List?) + ?.map((e) => (e as num).toDouble()) + .toList(); + + return SemanticsNodeModel( + id: json['id']?.toString() ?? '', + label: json['label']?.toString() ?? '', + value: json['value']?.toString() ?? '', + hint: json['hint']?.toString() ?? '', + tooltip: json['tooltip']?.toString() ?? '', + increasedValue: json['increasedValue']?.toString() ?? '', + decreasedValue: json['decreasedValue']?.toString() ?? '', + flags: flags, + actions: actions, + widgetName: json['widgetName']?.toString() ?? '', + rectString: rectString, + transform: transform, + ); + } + @override void dispose() { + unawaited(_disposeSemanticsOnApp()); brightness.dispose(); textScale.dispose(); boldText.dispose(); screenReader.dispose(); highContrast.dispose(); + semanticsRoots.dispose(); + semanticsTreeLoading.dispose(); + semanticsTreeError.dispose(); super.dispose(); } + + Future _disposeSemanticsOnApp() async { + try { + if (serviceConnection.serviceManager.connectedState.value.connected) { + await serviceConnection.serviceManager + .callServiceExtensionOnMainIsolate( + 'ext.flutter.accessibility.disposeSemantics', + ); + } + } catch (_) { + // Ignore errors if the app or isolate connection is already closed. + } + } } diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart index 8081874954c..d8f54b2b93f 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart @@ -5,7 +5,11 @@ import 'package:devtools_app_shared/ui.dart'; import 'package:flutter/material.dart'; +import '../../shared/analytics/constants.dart' as gac; +import '../../shared/globals.dart'; import '../../shared/ui/common_widgets.dart'; +import '../../shared/ui/tree_view.dart'; +import 'accessibility_controller.dart'; /// A pane that displays the semantics tree of the connected app. class AccessibilitySemanticsTreePane extends StatelessWidget { @@ -13,17 +17,204 @@ class AccessibilitySemanticsTreePane extends StatelessWidget { @override Widget build(BuildContext context) { - return const DevToolsAreaPane( - header: AreaPaneHeader( - title: Text('Semantics Tree'), - includeTopBorder: false, - roundedTopBorder: false, + final controller = screenControllers.lookup(); + return ValueListenableBuilder>( + valueListenable: controller.semanticsRoots, + builder: (context, roots, _) { + return DevToolsAreaPane( + header: AreaPaneHeader( + title: const Text('Semantics Tree'), + includeTopBorder: false, + roundedTopBorder: false, + actions: [ + if (roots.isNotEmpty) + RefreshButton( + iconOnly: true, + tooltip: 'Refresh Semantics Tree', + gaScreen: gac.accessibility, + gaSelection: gac.refresh, + onPressed: controller.loadSemanticsTree, + ), + ], + ), + child: ValueListenableBuilder( + valueListenable: controller.semanticsTreeLoading, + builder: (context, loading, _) { + if (loading) { + return const CenteredCircularProgressIndicator(); + } + return ValueListenableBuilder( + valueListenable: controller.semanticsTreeError, + builder: (context, error, _) { + if (error != null) { + return _SemanticsTreeErrorState( + errorMessage: error, + onRetry: controller.loadSemanticsTree, + ); + } + if (roots.isEmpty) { + return _SemanticsTreeEmptyState( + onLoad: controller.loadSemanticsTree, + ); + } + return _SemanticsTreeContent(controller: controller); + }, + ); + }, + ), + ); + }, + ); + } +} + +class _SemanticsTreeEmptyState extends StatelessWidget { + const _SemanticsTreeEmptyState({required this.onLoad}); + + final VoidCallback onLoad; + + @override + Widget build(BuildContext context) { + return Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + const CenteredMessage( + message: + 'No semantics tree loaded. Inspect the accessibility hierarchy of the connected app.', + ), + const SizedBox(height: defaultSpacing), + DevToolsButton( + onPressed: onLoad, + icon: Icons.account_tree_outlined, + label: 'Load Semantics Tree', + elevated: true, + ), + ], ), - child: CenteredMessage( - message: - 'Accessibility semantics tree placeholder.\n' - '// TODO(hannah-hyj): Implement semantics tree view and details explorer.', + ); + } +} + +class _SemanticsTreeErrorState extends StatelessWidget { + const _SemanticsTreeErrorState({ + required this.errorMessage, + required this.onRetry, + }); + + final String errorMessage; + final VoidCallback onRetry; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + return Center( + child: Padding( + padding: const EdgeInsets.all(defaultSpacing), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + SelectableText( + errorMessage, + style: theme.regularTextStyle.copyWith( + color: theme.colorScheme.error, + ), + textAlign: TextAlign.center, + ), + const SizedBox(height: defaultSpacing), + DevToolsButton( + onPressed: onRetry, + icon: Icons.refresh, + label: 'Try Again', + elevated: true, + ), + ], + ), ), ); } } + +class _SemanticsTreeContent extends StatelessWidget { + const _SemanticsTreeContent({required this.controller}); + + final AccessibilityController controller; + + static IconData _iconForNode(SemanticsNodeModel node) { + if (node.flags.contains('isButton')) return Icons.smart_button_rounded; + if (node.flags.contains('isTextField')) return Icons.text_fields_rounded; + if (node.flags.contains('isHeader')) return Icons.title_rounded; + if (node.flags.contains('isSlider')) return Icons.linear_scale_rounded; + if (node.flags.contains('hasCheckedState')) return Icons.check_box_outlined; + return Icons.widgets_outlined; + } + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final colorScheme = theme.colorScheme; + + return TreeView( + dataRootsListenable: controller.semanticsRoots, + dataDisplayProvider: (node, onPressed) { + return InkWell( + onTap: onPressed, + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: denseSpacing), + child: Row( + children: [ + Icon( + _iconForNode(node), + size: defaultIconSize, + color: colorScheme.onSurface.withValues(alpha: 0.7), + ), + const SizedBox(width: denseSpacing), + Text( + 'SemanticsNode #${node.id}', + maxLines: 1, + style: theme.fixedFontStyle, + ), + if (node.label.isNotEmpty) ...[ + const SizedBox(width: denseSpacing), + Flexible( + child: Text( + '"${node.label}"', + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: theme.subtleTextStyle.copyWith( + fontStyle: FontStyle.italic, + color: colorScheme.onSurface.withValues(alpha: 0.6), + ), + ), + ), + ], + if (node.widgetName.isNotEmpty) ...[ + const SizedBox(width: denseSpacing), + Container( + padding: const EdgeInsets.symmetric( + horizontal: densePadding, + ), + decoration: BoxDecoration( + color: colorScheme.primaryContainer.withValues( + alpha: 0.2, + ), + borderRadius: BorderRadius.circular(4), + ), + child: Text( + node.widgetName, + maxLines: 1, + style: theme.subtleTextStyle.copyWith( + color: colorScheme.primary, + fontSize: 10, + ), + ), + ), + ], + ], + ), + ), + ); + }, + ); + } +} diff --git a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart index 1363f9d330c..0a0bd1d6826 100644 --- a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart +++ b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart @@ -8,8 +8,10 @@ library; import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_app_shared/utils.dart'; import 'package:devtools_test/devtools_test.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:mockito/mockito.dart'; +import 'package:vm_service/vm_service.dart'; void main() { group('AccessibilityController', () { @@ -24,6 +26,19 @@ void main() { fakeServiceConnection.serviceManager.connectedApp!.isProfileBuildNow, ).thenReturn(false); + fakeServiceConnection + .serviceManager + .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = + Response.parse({})!; + fakeServiceConnection + .serviceManager + .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = + Response.parse({ + 'data': { + '0': {'id': '0', 'label': 'Root'}, + }, + })!; + setGlobal(NotificationService, NotificationService()); setGlobal( DevToolsEnvironmentParameters, @@ -36,7 +51,17 @@ void main() { }); test('initial state', () { - expect(controller.brightness.value, BrightnessOverride.system); + final uninitializedController = AccessibilityController(); + expect( + uninitializedController.brightness.value, + BrightnessOverride.system, + ); + expect(uninitializedController.textScale.value, 1.0); + expect(uninitializedController.boldText.value, isFalse); + expect(uninitializedController.screenReader.value, isFalse); + expect(uninitializedController.highContrast.value, isFalse); + expect(uninitializedController.semanticsRoots.value, isEmpty); + expect(uninitializedController.semanticsTreeLoading.value, isFalse); }); test( @@ -114,5 +139,350 @@ void main() { expect(systemState.enabled, isFalse); }, ); + + test('SemanticsNodeModel properties and shallowCopy', () { + final child = SemanticsNodeModel( + id: '1', + label: 'Child Node', + value: '10', + hint: 'Double tap to activate', + tooltip: 'Child Tooltip', + increasedValue: '11', + decreasedValue: '9', + flags: ['isButton', 'hasCheckedState'], + actions: ['tap', 'increase'], + widgetName: 'ElevatedButton', + rectString: 'rect: Rect.fromLTWH(0, 0, 50, 20)', + transform: [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + ], + ); + final parent = SemanticsNodeModel( + id: '0', + label: 'Parent Node', + flags: ['isHeader'], + widgetName: 'Column', + rectString: 'rect: Rect.fromLTWH(0, 0, 100, 100)', + )..addChild(child); + + expect(parent.children, hasLength(1)); + expect(parent.children.first.id, equals('1')); + + final copy = child.shallowCopy(); + expect(copy.id, equals('1')); + expect(copy.label, equals('Child Node')); + expect(copy.value, equals('10')); + expect(copy.hint, equals('Double tap to activate')); + expect(copy.tooltip, equals('Child Tooltip')); + expect(copy.increasedValue, equals('11')); + expect(copy.decreasedValue, equals('9')); + expect(copy.flags, equals(['isButton', 'hasCheckedState'])); + expect(copy.actions, equals(['tap', 'increase'])); + expect(copy.widgetName, equals('ElevatedButton')); + expect(copy.rectString, equals('rect: Rect.fromLTWH(0, 0, 50, 20)')); + expect(copy.transform, hasLength(16)); + expect(copy.children, isEmpty); + }); + + test( + 'loadSemanticsTree sets error state when no main isolate connected', + () async { + final fakeServiceConnection = _NullIsolateServiceConnectionManager(); + setGlobal(ServiceConnectionManager, fakeServiceConnection); + + final testController = AccessibilityController(); + expect(testController.semanticsTreeError.value, isNull); + await testController.loadSemanticsTree(); + expect( + testController.semanticsTreeError.value, + equals('Failed to load semantics tree: no connected application.'), + ); + expect(testController.semanticsTreeLoading.value, isFalse); + expect(testController.semanticsRoots.value, isEmpty); + }, + ); + + test( + 'loadSemanticsTree sets error state when service extension returns error', + () async { + final fakeServiceManager = + serviceConnection.serviceManager as FakeServiceManager; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = + Response.parse({})!; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = + Response.parse({'error': 'Semantics not enabled.'})!; + + final testController = AccessibilityController(); + await testController.loadSemanticsTree(); + + expect( + testController.semanticsTreeError.value, + equals( + 'Failed to load semantics tree: Exception: Semantics not enabled.', + ), + ); + expect(testController.semanticsRoots.value, isEmpty); + }, + ); + + test( + 'loadSemanticsTree parses full SemanticsNode.toJson format with multiple nodes', + () async { + final fakeServiceManager = + serviceConnection.serviceManager as FakeServiceManager; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = + Response.parse({})!; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = + Response.parse({ + 'data': { + '0': { + 'id': 0, + 'label': 'Root View', + 'value': 'Main Screen', + 'hint': '', + 'tooltip': '', + 'increasedValue': '', + 'decreasedValue': '', + 'flags': ['hasEnabledState', 'isEnabled'], + 'actions': [], + 'rect': { + 'left': 0.0, + 'top': 0.0, + 'width': 390.0, + 'height': 844.0, + }, + 'transform': [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + ], + 'childrenInTraversalOrder': [1, 2], + 'childrenInHitTestOrder': [2, 1], + }, + '1': { + 'id': 1, + 'label': 'Settings Header', + 'flags': ['isHeader'], + 'actions': [], + 'rect': { + 'left': 16.0, + 'top': 40.0, + 'width': 358.0, + 'height': 32.0, + }, + }, + '2': { + 'id': 2, + 'label': 'Search Input', + 'value': 'Flutter', + 'hint': 'Enter search query', + 'tooltip': 'Search field', + 'flags': ['isTextField'], + 'actions': ['tap', 'setSelection'], + 'rect': { + 'left': 16.0, + 'top': 88.0, + 'width': 358.0, + 'height': 48.0, + }, + 'childrenInTraversalOrder': [3], + 'childrenInHitTestOrder': [3], + }, + '3': { + 'id': 3, + 'label': 'Clear Text', + 'tooltip': 'Clear input content', + 'flags': ['isButton', 'hasCheckedState'], + 'actions': ['tap'], + 'rect': { + 'left': 330.0, + 'top': 96.0, + 'width': 32.0, + 'height': 32.0, + }, + }, + }, + })!; + + final testController = AccessibilityController(); + await testController.loadSemanticsTree(); + + expect(testController.semanticsRoots.value, hasLength(1)); + final root = testController.semanticsRoots.value.first; + expect(root.id, equals('0')); + expect(root.label, equals('Root View')); + expect(root.value, equals('Main Screen')); + expect(root.flags, equals(['hasEnabledState', 'isEnabled'])); + expect( + root.rectString, + equals('rect: Rect.fromLTWH(0.0, 0.0, 390.0, 844.0)'), + ); + expect(root.transform, hasLength(16)); + expect(root.children, hasLength(2)); + + // Node 1: Settings Header + final headerNode = root.children[0]; + expect(headerNode.id, equals('1')); + expect(headerNode.label, equals('Settings Header')); + expect(headerNode.flags, equals(['isHeader'])); + expect( + headerNode.rectString, + equals('rect: Rect.fromLTWH(16.0, 40.0, 358.0, 32.0)'), + ); + expect(headerNode.children, isEmpty); + + // Node 2: Search Input + final searchNode = root.children[1]; + expect(searchNode.id, equals('2')); + expect(searchNode.label, equals('Search Input')); + expect(searchNode.value, equals('Flutter')); + expect(searchNode.hint, equals('Enter search query')); + expect(searchNode.tooltip, equals('Search field')); + expect(searchNode.flags, equals(['isTextField'])); + expect(searchNode.actions, equals(['tap', 'setSelection'])); + expect(searchNode.children, hasLength(1)); + + // Node 3: Clear Text Button (child of Node 2) + final clearButtonNode = searchNode.children.first; + expect(clearButtonNode.id, equals('3')); + expect(clearButtonNode.label, equals('Clear Text')); + expect(clearButtonNode.tooltip, equals('Clear input content')); + expect(clearButtonNode.flags, equals(['isButton', 'hasCheckedState'])); + expect(clearButtonNode.actions, equals(['tap'])); + expect( + clearButtonNode.rectString, + equals('rect: Rect.fromLTWH(330.0, 96.0, 32.0, 32.0)'), + ); + expect(clearButtonNode.children, isEmpty); + }, + ); + + test( + 'loadSemanticsTree parses flat nodes map and builds child hierarchy', + () async { + final fakeServiceManager = + serviceConnection.serviceManager as FakeServiceManager; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = + Response.parse({})!; + fakeServiceManager + .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = + Response.parse({ + 'data': { + '0': { + 'id': 0, + 'label': 'Root', + 'childrenInTraversalOrder': [1], + }, + '1': { + 'id': 1, + 'label': 'Child full', + 'flags': ['isButton'], + }, + }, + })!; + + final testController = AccessibilityController(); + await testController.loadSemanticsTree(); + + expect(testController.semanticsRoots.value, hasLength(1)); + final root = testController.semanticsRoots.value.first; + expect(root.id, equals('0')); + expect(root.label, equals('Root')); + expect(root.children, hasLength(1)); + expect(root.children.first.id, equals('1')); + expect(root.children.first.label, equals('Child full')); + expect(root.children.first.flags, equals(['isButton'])); + }, + ); + + test('dispose calls disposeSemantics', () async { + final recordingServiceConnection = _RecordingServiceConnectionManager(); + setGlobal(ServiceConnectionManager, recordingServiceConnection); + + final testController = AccessibilityController(); + testController.dispose(); + await Future.delayed(Duration.zero); + + final calls = + (recordingServiceConnection.serviceManager + as _RecordingServiceManager) + .recordedCalls; + expect( + calls.any( + (call) => call.$1 == 'ext.flutter.accessibility.disposeSemantics', + ), + isTrue, + ); + }); }); } + +class _RecordingServiceConnectionManager extends FakeServiceConnectionManager { + @override + late final FakeServiceManager serviceManager = _RecordingServiceManager(); +} + +// ignore: subtype_of_sealed_class +class _RecordingServiceManager extends FakeServiceManager { + final recordedCalls = <(String, Map?)>[]; + + @override + Future callServiceExtensionOnMainIsolate( + String method, { + Map? args, + }) async { + recordedCalls.add((method, args)); + return serviceExtensionResponses[method] ?? Response.parse({})!; + } +} + +class _NullIsolateServiceConnectionManager + extends FakeServiceConnectionManager { + @override + late final FakeServiceManager serviceManager = _NullIsolateServiceManager(); +} + +// ignore: subtype_of_sealed_class +class _NullIsolateServiceManager extends FakeServiceManager { + @override + late final FakeIsolateManager isolateManager = _NullIsolateManager(); +} + +base class _NullIsolateManager extends FakeIsolateManager { + @override + ValueListenable get mainIsolate => + ValueNotifier(null); +} diff --git a/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart b/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart index ba5e53e637b..9104d9b0a2d 100644 --- a/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart +++ b/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart @@ -181,5 +181,129 @@ void main() { expect(controller.highContrast.value, isTrue); }, ); + + testWidgetsWithWindowSize( + 'renders semantics tree when nodes are loaded', + windowSize, + (WidgetTester tester) async { + final rootNode = SemanticsNodeModel( + id: '0', + label: 'Root Node', + flags: ['isHeader'], + widgetName: 'HeaderWidget', + rectString: 'rect: Rect.fromLTWH(0, 0, 100, 50)', + ); + controller.semanticsRoots.value = [rootNode]; + + await pumpAccessibilityScreen(tester); + await tester.pumpAndSettle(); + + expect(find.text('SemanticsNode #0'), findsAtLeastNWidgets(1)); + expect(find.text('"Root Node"'), findsAtLeastNWidgets(1)); + expect(find.text('HeaderWidget'), findsOneWidget); + }, + ); + + testWidgetsWithWindowSize( + 'renders semantics tree error state when error occurs', + windowSize, + (WidgetTester tester) async { + controller.semanticsTreeError.value = + 'Failed to load semantics tree: network error.'; + + await pumpAccessibilityScreen(tester); + await tester.pumpAndSettle(); + + expect( + find.text('Failed to load semantics tree: network error.'), + findsOneWidget, + ); + expect(find.text('Try Again'), findsOneWidget); + }, + ); + + testWidgetsWithWindowSize( + 'renders parent and child nodes in semantics tree', + windowSize, + (WidgetTester tester) async { + final childNode = SemanticsNodeModel( + id: '1', + label: 'Child Node', + flags: ['isButton'], + widgetName: 'ElevatedButton', + ); + final rootNode = SemanticsNodeModel( + id: '0', + label: 'Root Node', + widgetName: 'Column', + )..addChild(childNode); + + rootNode.expandCascading(); + controller.semanticsRoots.value = [rootNode]; + + await pumpAccessibilityScreen(tester); + await tester.pumpAndSettle(); + + expect(find.text('SemanticsNode #0'), findsOneWidget); + expect(find.text('SemanticsNode #1'), findsOneWidget); + expect(find.text('"Child Node"'), findsOneWidget); + expect(find.text('ElevatedButton'), findsOneWidget); + }, + ); + + testWidgetsWithWindowSize( + 'renders empty state with Load Semantics Tree button when no roots', + windowSize, + (WidgetTester tester) async { + controller.semanticsRoots.value = []; + await pumpAccessibilityScreen(tester); + controller.semanticsTreeError.value = null; + await tester.pumpAndSettle(); + + expect( + find.text( + 'No semantics tree loaded. Inspect the accessibility hierarchy of the connected app.', + ), + findsOneWidget, + ); + expect( + find.widgetWithText(DevToolsButton, 'Load Semantics Tree'), + findsOneWidget, + ); + expect(find.byType(RefreshButton), findsNothing); + }, + ); + + testWidgetsWithWindowSize( + 'renders refresh button when roots are loaded', + windowSize, + (WidgetTester tester) async { + final rootNode = SemanticsNodeModel(id: '0', label: 'Root Node'); + controller.semanticsRoots.value = [rootNode]; + + await pumpAccessibilityScreen(tester); + await tester.pumpAndSettle(); + + expect(find.byType(RefreshButton), findsOneWidget); + }, + ); + + testWidgetsWithWindowSize( + 'renders appropriate icon for node with hasCheckedState flag', + windowSize, + (WidgetTester tester) async { + final node = SemanticsNodeModel( + id: '0', + label: 'Checkbox Node', + flags: ['hasCheckedState'], + ); + controller.semanticsRoots.value = [node]; + + await pumpAccessibilityScreen(tester); + await tester.pumpAndSettle(); + + expect(find.byIcon(Icons.check_box_outlined), findsOneWidget); + }, + ); }); } From 9363bdfa40a0ae77e91bebe65de73078a395e216 Mon Sep 17 00:00:00 2001 From: hangyu Date: Fri, 28 Aug 2026 11:24:53 -0700 Subject: [PATCH 2/7] lint --- .../accessibility/accessibility_controller_test.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart index 0a0bd1d6826..fc4f5d67b7e 100644 --- a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart +++ b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart @@ -452,10 +452,10 @@ void main() { class _RecordingServiceConnectionManager extends FakeServiceConnectionManager { @override - late final FakeServiceManager serviceManager = _RecordingServiceManager(); + late final serviceManager = _RecordingServiceManager(); } -// ignore: subtype_of_sealed_class +// ignore: subtype_of_sealed_class, fake for testing. class _RecordingServiceManager extends FakeServiceManager { final recordedCalls = <(String, Map?)>[]; @@ -472,13 +472,13 @@ class _RecordingServiceManager extends FakeServiceManager { class _NullIsolateServiceConnectionManager extends FakeServiceConnectionManager { @override - late final FakeServiceManager serviceManager = _NullIsolateServiceManager(); + late final serviceManager = _NullIsolateServiceManager(); } -// ignore: subtype_of_sealed_class +// ignore: subtype_of_sealed_class, fake for testing. class _NullIsolateServiceManager extends FakeServiceManager { @override - late final FakeIsolateManager isolateManager = _NullIsolateManager(); + late final isolateManager = _NullIsolateManager(); } base class _NullIsolateManager extends FakeIsolateManager { From 70073d39f5a586927feb2b47a192178d326b996b Mon Sep 17 00:00:00 2001 From: hangyu Date: Fri, 28 Aug 2026 12:58:48 -0700 Subject: [PATCH 3/7] fix lint --- .../accessibility/accessibility_controller.dart | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index 74ff08a6159..b4424c52429 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -41,33 +41,49 @@ class SemanticsNodeModel extends TreeNode { final String label; /// The current value of this node (e.g. the text in a text field) (maps to [SemanticsData.value]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String value; /// Additional hint text spoken after a delay (maps to [SemanticsData.hint]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String hint; /// A brief description of the widget the semantics node represents (maps to [SemanticsData.tooltip]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String tooltip; /// The value that the node will take if the user increases it (maps to [SemanticsData.increasedValue]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String increasedValue; /// The value that the node will take if the user decreases it (maps to [SemanticsData.decreasedValue]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String decreasedValue; /// Semantic flags active on this node (e.g. `'isButton'`, `'isHeader'`). final List flags; /// Semantic actions that can be performed on this node (e.g. `'tap'`, `'scrollLeft'`). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final List actions; /// The name of the Flutter widget that produced this node, if available. final String widgetName; /// Human-readable representation of this node's bounding rect (maps to [SemanticsData.rect]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final String rectString; /// The transformation matrix to apply to this node's coordinate system (maps to [SemanticsData.transform]). + // TODO(hangyujin): Display in node details UI. + // ignore: unused-code, will be displayed when node details UI is added. final List? transform; @override From 75f07e270840d362a3313bc033e02c41185302f5 Mon Sep 17 00:00:00 2001 From: hangyu Date: Mon, 31 Aug 2026 14:56:20 -0700 Subject: [PATCH 4/7] resolve comments --- packages/devtools_app/lib/devtools_app.dart | 1 + .../accessibility_controller.dart | 130 +------- .../accessibility/semantics_node_model.dart | 56 ++++ .../accessibility/semantics_tree_pane.dart | 24 +- .../src/service/service_registrations.dart | 9 + .../accessibility_controller_test.dart | 309 ++++++++---------- .../accessibility_screen_test.dart | 9 +- 7 files changed, 234 insertions(+), 304 deletions(-) create mode 100644 packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart diff --git a/packages/devtools_app/lib/devtools_app.dart b/packages/devtools_app/lib/devtools_app.dart index 80880547987..5fd07a4ff1e 100644 --- a/packages/devtools_app/lib/devtools_app.dart +++ b/packages/devtools_app/lib/devtools_app.dart @@ -12,6 +12,7 @@ export 'src/framework/scaffold/app_bar.dart'; export 'src/framework/scaffold/status_line.dart'; export 'src/screens/accessibility/accessibility_controller.dart'; export 'src/screens/accessibility/accessibility_screen.dart'; +export 'src/screens/accessibility/semantics_node_model.dart'; export 'src/screens/app_size/app_size_controller.dart'; export 'src/screens/app_size/app_size_screen.dart'; export 'src/screens/debugger/breakpoint_manager.dart'; diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index b4424c52429..6b910f85e89 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -2,108 +2,23 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. -/// @docImport 'package:flutter/semantics.dart'; -library; - import 'dart:async'; import 'package:devtools_app_shared/service.dart'; import 'package:devtools_app_shared/utils.dart'; import 'package:flutter/foundation.dart'; +import 'package:logging/logging.dart'; import '../../service/service_extensions.dart' as extensions; +import '../../service/service_registrations.dart' as registrations; import '../../shared/framework/screen.dart'; import '../../shared/framework/screen_controllers.dart'; import '../../shared/globals.dart'; -import '../../shared/primitives/trees.dart'; - -/// Represents a node in the semantics tree. -class SemanticsNodeModel extends TreeNode { - SemanticsNodeModel({ - required this.id, - this.label = '', - this.value = '', - this.hint = '', - this.tooltip = '', - this.increasedValue = '', - this.decreasedValue = '', - this.flags = const [], - this.actions = const [], - this.widgetName = '', - this.rectString = '', - this.transform, - }); - - /// The semantics node identifier, as provided by the Flutter framework. - final String id; - - /// The user-visible label announced by screen readers (maps to [SemanticsData.label]). - final String label; - - /// The current value of this node (e.g. the text in a text field) (maps to [SemanticsData.value]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String value; - - /// Additional hint text spoken after a delay (maps to [SemanticsData.hint]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String hint; - - /// A brief description of the widget the semantics node represents (maps to [SemanticsData.tooltip]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String tooltip; - - /// The value that the node will take if the user increases it (maps to [SemanticsData.increasedValue]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String increasedValue; - - /// The value that the node will take if the user decreases it (maps to [SemanticsData.decreasedValue]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String decreasedValue; - - /// Semantic flags active on this node (e.g. `'isButton'`, `'isHeader'`). - final List flags; - - /// Semantic actions that can be performed on this node (e.g. `'tap'`, `'scrollLeft'`). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final List actions; - - /// The name of the Flutter widget that produced this node, if available. - final String widgetName; +import 'semantics_node_model.dart'; - /// Human-readable representation of this node's bounding rect (maps to [SemanticsData.rect]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final String rectString; +export 'semantics_node_model.dart'; - /// The transformation matrix to apply to this node's coordinate system (maps to [SemanticsData.transform]). - // TODO(hangyujin): Display in node details UI. - // ignore: unused-code, will be displayed when node details UI is added. - final List? transform; - - @override - SemanticsNodeModel shallowCopy() { - return SemanticsNodeModel( - id: id, - label: label, - value: value, - hint: hint, - tooltip: tooltip, - increasedValue: increasedValue, - decreasedValue: decreasedValue, - flags: flags, - actions: actions, - widgetName: widgetName, - rectString: rectString, - transform: transform, - ); - } -} +final _log = Logger('accessibility_controller'); /// Modes for brightness override in the accessibility controls. enum BrightnessOverride { @@ -250,19 +165,15 @@ class AccessibilityController extends DevToolsScreenController semanticsTreeLoading.value = true; semanticsTreeError.value = null; - // Intentionally do NOT clear semanticsRoots here so that the old tree - // remains visible while a refresh is in flight. try { await serviceConnection.serviceManager.callServiceExtensionOnMainIsolate( - 'ext.flutter.accessibility.enableSemantics', + registrations.enableSemantics, args: {'enabled': 'true'}, ); final response = await serviceConnection.serviceManager - .callServiceExtensionOnMainIsolate( - 'ext.flutter.accessibility.getSemanticsTree', - ); + .callServiceExtensionOnMainIsolate(registrations.getSemanticsTree); final json = response.json; if (json != null && json.containsKey('error')) { @@ -296,8 +207,7 @@ class AccessibilityController extends DevToolsScreenController semanticsRoots.value = roots; semanticsTreeError.value = null; } catch (e, st) { - debugPrint('Error loading semantics tree: $e'); - debugPrint('$st'); + _log.warning('Error loading semantics tree: $e', e, st); semanticsRoots.value = []; semanticsTreeError.value = 'Failed to load semantics tree: $e'; } finally { @@ -339,30 +249,14 @@ class AccessibilityController extends DevToolsScreenController } SemanticsNodeModel _parseSemanticsNode(Map json) { - final rect = json['rect'] as Map?; - final rectString = rect != null - ? 'rect: Rect.fromLTWH(${rect['left']}, ${rect['top']}, ${rect['width']}, ${rect['height']})' - : 'Rect.zero'; - - final flags = (json['flags'] as List?)?.cast() ?? const []; - final actions = (json['actions'] as List?)?.cast() ?? const []; - final transform = (json['transform'] as List?) - ?.map((e) => (e as num).toDouble()) - .toList(); + final rawFlags = json['flags'] as List?; + final flags = SemanticsNodeModel.parseFlags(rawFlags); return SemanticsNodeModel( id: json['id']?.toString() ?? '', label: json['label']?.toString() ?? '', - value: json['value']?.toString() ?? '', - hint: json['hint']?.toString() ?? '', - tooltip: json['tooltip']?.toString() ?? '', - increasedValue: json['increasedValue']?.toString() ?? '', - decreasedValue: json['decreasedValue']?.toString() ?? '', flags: flags, - actions: actions, widgetName: json['widgetName']?.toString() ?? '', - rectString: rectString, - transform: transform, ); } @@ -384,9 +278,7 @@ class AccessibilityController extends DevToolsScreenController try { if (serviceConnection.serviceManager.connectedState.value.connected) { await serviceConnection.serviceManager - .callServiceExtensionOnMainIsolate( - 'ext.flutter.accessibility.disposeSemantics', - ); + .callServiceExtensionOnMainIsolate(registrations.disposeSemantics); } } catch (_) { // Ignore errors if the app or isolate connection is already closed. diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart new file mode 100644 index 00000000000..12b356725be --- /dev/null +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart @@ -0,0 +1,56 @@ +// Copyright 2026 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. + +/// @docImport 'package:flutter/semantics.dart'; +library; + +import 'dart:ui' show SemanticsFlag; + +import '../../shared/primitives/trees.dart'; + +/// Represents a node in the accessibility semantics tree. +class SemanticsNodeModel extends TreeNode { + SemanticsNodeModel({ + required this.id, + this.label = '', + this.flags = const {}, + this.widgetName = '', + }); + + /// The semantics node identifier, as provided by the Flutter framework. + final String id; + + /// The user-visible label announced by screen readers (maps to [SemanticsData.label]). + final String label; + + /// Semantic flags active on this node. + final Set flags; + + /// The name of the Flutter widget that produced this node, if available. + final String widgetName; + + /// Mapping from flag name strings to [SemanticsFlag] instances. + static final _flagByName = { + for (final flag in SemanticsFlag.values) flag.name: flag, + }; + + /// Parses a list of flag name strings into a set of [SemanticsFlag]s. + static Set parseFlags(List? rawFlags) { + if (rawFlags == null) return const {}; + return rawFlags + .map((e) => _flagByName[e.toString()]) + .whereType() + .toSet(); + } + + @override + SemanticsNodeModel shallowCopy() { + return SemanticsNodeModel( + id: id, + label: label, + flags: flags, + widgetName: widgetName, + ); + } +} diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart index d8f54b2b93f..cc8c3e0e16a 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd. +import 'dart:ui' show SemanticsFlag; + import 'package:devtools_app_shared/ui.dart'; import 'package:flutter/material.dart'; @@ -141,11 +143,21 @@ class _SemanticsTreeContent extends StatelessWidget { final AccessibilityController controller; static IconData _iconForNode(SemanticsNodeModel node) { - if (node.flags.contains('isButton')) return Icons.smart_button_rounded; - if (node.flags.contains('isTextField')) return Icons.text_fields_rounded; - if (node.flags.contains('isHeader')) return Icons.title_rounded; - if (node.flags.contains('isSlider')) return Icons.linear_scale_rounded; - if (node.flags.contains('hasCheckedState')) return Icons.check_box_outlined; + if (node.flags.contains(SemanticsFlag.isButton)) { + return Icons.smart_button_rounded; + } + if (node.flags.contains(SemanticsFlag.isTextField)) { + return Icons.text_fields_rounded; + } + if (node.flags.contains(SemanticsFlag.isHeader)) { + return Icons.title_rounded; + } + if (node.flags.contains(SemanticsFlag.isSlider)) { + return Icons.linear_scale_rounded; + } + if (node.flags.contains(SemanticsFlag.hasCheckedState)) { + return Icons.check_box_outlined; + } return Icons.widgets_outlined; } @@ -205,7 +217,7 @@ class _SemanticsTreeContent extends StatelessWidget { maxLines: 1, style: theme.subtleTextStyle.copyWith( color: colorScheme.primary, - fontSize: 10, + fontSize: smallFontSize, ), ), ), diff --git a/packages/devtools_app/lib/src/service/service_registrations.dart b/packages/devtools_app/lib/src/service/service_registrations.dart index 2f6c0bc0ce0..958e071cae3 100644 --- a/packages/devtools_app/lib/src/service/service_registrations.dart +++ b/packages/devtools_app/lib/src/service/service_registrations.dart @@ -60,3 +60,12 @@ const dwdsSendEvent = 'ext.dwds.sendEvent'; /// Service extension that returns whether or not the Impeller rendering engine /// is being used (if false, the app is using SKIA). const isImpellerEnabled = 'ext.ui.window.impellerEnabled'; + +/// Service extension to enable or disable semantics on the connected app. +const enableSemantics = 'ext.flutter.accessibility.enableSemantics'; + +/// Service extension to fetch the accessibility semantics tree from the connected app. +const getSemanticsTree = 'ext.flutter.accessibility.getSemanticsTree'; + +/// Service extension to dispose semantics on the connected app. +const disposeSemantics = 'ext.flutter.accessibility.disposeSemantics'; diff --git a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart index fc4f5d67b7e..06f3eb8f808 100644 --- a/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart +++ b/packages/devtools_app/test/screens/accessibility/accessibility_controller_test.dart @@ -5,7 +5,11 @@ @TestOn('vm') library; +import 'dart:ui' show SemanticsFlag; + import 'package:devtools_app/devtools_app.dart'; +import 'package:devtools_app/src/service/service_registrations.dart' + as registrations; import 'package:devtools_app_shared/utils.dart'; import 'package:devtools_test/devtools_test.dart'; import 'package:flutter/foundation.dart'; @@ -28,11 +32,11 @@ void main() { fakeServiceConnection .serviceManager - .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = + .serviceExtensionResponses[registrations.enableSemantics] = Response.parse({})!; fakeServiceConnection .serviceManager - .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = + .serviceExtensionResponses[registrations.getSemanticsTree] = Response.parse({ 'data': { '0': {'id': '0', 'label': 'Root'}, @@ -144,40 +148,14 @@ void main() { final child = SemanticsNodeModel( id: '1', label: 'Child Node', - value: '10', - hint: 'Double tap to activate', - tooltip: 'Child Tooltip', - increasedValue: '11', - decreasedValue: '9', - flags: ['isButton', 'hasCheckedState'], - actions: ['tap', 'increase'], + flags: {SemanticsFlag.isButton, SemanticsFlag.hasCheckedState}, widgetName: 'ElevatedButton', - rectString: 'rect: Rect.fromLTWH(0, 0, 50, 20)', - transform: [ - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - ], ); final parent = SemanticsNodeModel( id: '0', label: 'Parent Node', - flags: ['isHeader'], + flags: {SemanticsFlag.isHeader}, widgetName: 'Column', - rectString: 'rect: Rect.fromLTWH(0, 0, 100, 100)', )..addChild(child); expect(parent.children, hasLength(1)); @@ -186,16 +164,11 @@ void main() { final copy = child.shallowCopy(); expect(copy.id, equals('1')); expect(copy.label, equals('Child Node')); - expect(copy.value, equals('10')); - expect(copy.hint, equals('Double tap to activate')); - expect(copy.tooltip, equals('Child Tooltip')); - expect(copy.increasedValue, equals('11')); - expect(copy.decreasedValue, equals('9')); - expect(copy.flags, equals(['isButton', 'hasCheckedState'])); - expect(copy.actions, equals(['tap', 'increase'])); + expect( + copy.flags, + equals({SemanticsFlag.isButton, SemanticsFlag.hasCheckedState}), + ); expect(copy.widgetName, equals('ElevatedButton')); - expect(copy.rectString, equals('rect: Rect.fromLTWH(0, 0, 50, 20)')); - expect(copy.transform, hasLength(16)); expect(copy.children, isEmpty); }); @@ -222,12 +195,14 @@ void main() { () async { final fakeServiceManager = serviceConnection.serviceManager as FakeServiceManager; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = - Response.parse({})!; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = - Response.parse({'error': 'Semantics not enabled.'})!; + fakeServiceManager.serviceExtensionResponses[registrations + .enableSemantics] = Response.parse( + {}, + )!; + fakeServiceManager.serviceExtensionResponses[registrations + .getSemanticsTree] = Response.parse({ + 'error': 'Semantics not enabled.', + })!; final testController = AccessibilityController(); await testController.loadSemanticsTree(); @@ -247,94 +222,94 @@ void main() { () async { final fakeServiceManager = serviceConnection.serviceManager as FakeServiceManager; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = - Response.parse({})!; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = - Response.parse({ - 'data': { - '0': { - 'id': 0, - 'label': 'Root View', - 'value': 'Main Screen', - 'hint': '', - 'tooltip': '', - 'increasedValue': '', - 'decreasedValue': '', - 'flags': ['hasEnabledState', 'isEnabled'], - 'actions': [], - 'rect': { - 'left': 0.0, - 'top': 0.0, - 'width': 390.0, - 'height': 844.0, - }, - 'transform': [ - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0, - ], - 'childrenInTraversalOrder': [1, 2], - 'childrenInHitTestOrder': [2, 1], - }, - '1': { - 'id': 1, - 'label': 'Settings Header', - 'flags': ['isHeader'], - 'actions': [], - 'rect': { - 'left': 16.0, - 'top': 40.0, - 'width': 358.0, - 'height': 32.0, - }, - }, - '2': { - 'id': 2, - 'label': 'Search Input', - 'value': 'Flutter', - 'hint': 'Enter search query', - 'tooltip': 'Search field', - 'flags': ['isTextField'], - 'actions': ['tap', 'setSelection'], - 'rect': { - 'left': 16.0, - 'top': 88.0, - 'width': 358.0, - 'height': 48.0, - }, - 'childrenInTraversalOrder': [3], - 'childrenInHitTestOrder': [3], - }, - '3': { - 'id': 3, - 'label': 'Clear Text', - 'tooltip': 'Clear input content', - 'flags': ['isButton', 'hasCheckedState'], - 'actions': ['tap'], - 'rect': { - 'left': 330.0, - 'top': 96.0, - 'width': 32.0, - 'height': 32.0, - }, - }, + fakeServiceManager.serviceExtensionResponses[registrations + .enableSemantics] = Response.parse( + {}, + )!; + fakeServiceManager.serviceExtensionResponses[registrations + .getSemanticsTree] = Response.parse({ + 'data': { + '0': { + 'id': 0, + 'label': 'Root View', + 'value': 'Main Screen', + 'hint': '', + 'tooltip': '', + 'increasedValue': '', + 'decreasedValue': '', + 'flags': ['hasEnabledState', 'isEnabled'], + 'actions': [], + 'rect': { + 'left': 0.0, + 'top': 0.0, + 'width': 390.0, + 'height': 844.0, + }, + 'transform': [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + ], + 'childrenInTraversalOrder': [1, 2], + 'childrenInHitTestOrder': [2, 1], + }, + '1': { + 'id': 1, + 'label': 'Settings Header', + 'flags': ['isHeader'], + 'actions': [], + 'rect': { + 'left': 16.0, + 'top': 40.0, + 'width': 358.0, + 'height': 32.0, + }, + }, + '2': { + 'id': 2, + 'label': 'Search Input', + 'value': 'Flutter', + 'hint': 'Enter search query', + 'tooltip': 'Search field', + 'flags': ['isTextField'], + 'actions': ['tap', 'setSelection'], + 'rect': { + 'left': 16.0, + 'top': 88.0, + 'width': 358.0, + 'height': 48.0, }, - })!; + 'childrenInTraversalOrder': [3], + 'childrenInHitTestOrder': [3], + }, + '3': { + 'id': 3, + 'label': 'Clear Text', + 'tooltip': 'Clear input content', + 'flags': ['isButton', 'hasCheckedState'], + 'actions': ['tap'], + 'rect': { + 'left': 330.0, + 'top': 96.0, + 'width': 32.0, + 'height': 32.0, + }, + }, + }, + })!; final testController = AccessibilityController(); await testController.loadSemanticsTree(); @@ -343,47 +318,33 @@ void main() { final root = testController.semanticsRoots.value.first; expect(root.id, equals('0')); expect(root.label, equals('Root View')); - expect(root.value, equals('Main Screen')); - expect(root.flags, equals(['hasEnabledState', 'isEnabled'])); expect( - root.rectString, - equals('rect: Rect.fromLTWH(0.0, 0.0, 390.0, 844.0)'), + root.flags, + equals({SemanticsFlag.hasEnabledState, SemanticsFlag.isEnabled}), ); - expect(root.transform, hasLength(16)); expect(root.children, hasLength(2)); // Node 1: Settings Header final headerNode = root.children[0]; expect(headerNode.id, equals('1')); expect(headerNode.label, equals('Settings Header')); - expect(headerNode.flags, equals(['isHeader'])); - expect( - headerNode.rectString, - equals('rect: Rect.fromLTWH(16.0, 40.0, 358.0, 32.0)'), - ); + expect(headerNode.flags, equals({SemanticsFlag.isHeader})); expect(headerNode.children, isEmpty); // Node 2: Search Input final searchNode = root.children[1]; expect(searchNode.id, equals('2')); expect(searchNode.label, equals('Search Input')); - expect(searchNode.value, equals('Flutter')); - expect(searchNode.hint, equals('Enter search query')); - expect(searchNode.tooltip, equals('Search field')); - expect(searchNode.flags, equals(['isTextField'])); - expect(searchNode.actions, equals(['tap', 'setSelection'])); + expect(searchNode.flags, equals({SemanticsFlag.isTextField})); expect(searchNode.children, hasLength(1)); // Node 3: Clear Text Button (child of Node 2) final clearButtonNode = searchNode.children.first; expect(clearButtonNode.id, equals('3')); expect(clearButtonNode.label, equals('Clear Text')); - expect(clearButtonNode.tooltip, equals('Clear input content')); - expect(clearButtonNode.flags, equals(['isButton', 'hasCheckedState'])); - expect(clearButtonNode.actions, equals(['tap'])); expect( - clearButtonNode.rectString, - equals('rect: Rect.fromLTWH(330.0, 96.0, 32.0, 32.0)'), + clearButtonNode.flags, + equals({SemanticsFlag.isButton, SemanticsFlag.hasCheckedState}), ); expect(clearButtonNode.children, isEmpty); }, @@ -394,25 +355,25 @@ void main() { () async { final fakeServiceManager = serviceConnection.serviceManager as FakeServiceManager; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.enableSemantics'] = - Response.parse({})!; - fakeServiceManager - .serviceExtensionResponses['ext.flutter.accessibility.getSemanticsTree'] = - Response.parse({ - 'data': { - '0': { - 'id': 0, - 'label': 'Root', - 'childrenInTraversalOrder': [1], - }, - '1': { - 'id': 1, - 'label': 'Child full', - 'flags': ['isButton'], - }, - }, - })!; + fakeServiceManager.serviceExtensionResponses[registrations + .enableSemantics] = Response.parse( + {}, + )!; + fakeServiceManager.serviceExtensionResponses[registrations + .getSemanticsTree] = Response.parse({ + 'data': { + '0': { + 'id': 0, + 'label': 'Root', + 'childrenInTraversalOrder': [1], + }, + '1': { + 'id': 1, + 'label': 'Child full', + 'flags': ['isButton'], + }, + }, + })!; final testController = AccessibilityController(); await testController.loadSemanticsTree(); @@ -424,7 +385,7 @@ void main() { expect(root.children, hasLength(1)); expect(root.children.first.id, equals('1')); expect(root.children.first.label, equals('Child full')); - expect(root.children.first.flags, equals(['isButton'])); + expect(root.children.first.flags, equals({SemanticsFlag.isButton})); }, ); @@ -441,9 +402,7 @@ void main() { as _RecordingServiceManager) .recordedCalls; expect( - calls.any( - (call) => call.$1 == 'ext.flutter.accessibility.disposeSemantics', - ), + calls.any((call) => call.$1 == registrations.disposeSemantics), isTrue, ); }); diff --git a/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart b/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart index 9104d9b0a2d..337d24e521c 100644 --- a/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart +++ b/packages/devtools_app/test/screens/accessibility/accessibility_screen_test.dart @@ -5,6 +5,8 @@ @TestOn('vm') library; +import 'dart:ui' show SemanticsFlag; + import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_app_shared/ui.dart'; import 'package:devtools_app_shared/utils.dart'; @@ -189,9 +191,8 @@ void main() { final rootNode = SemanticsNodeModel( id: '0', label: 'Root Node', - flags: ['isHeader'], + flags: {SemanticsFlag.isHeader}, widgetName: 'HeaderWidget', - rectString: 'rect: Rect.fromLTWH(0, 0, 100, 50)', ); controller.semanticsRoots.value = [rootNode]; @@ -229,7 +230,7 @@ void main() { final childNode = SemanticsNodeModel( id: '1', label: 'Child Node', - flags: ['isButton'], + flags: {SemanticsFlag.isButton}, widgetName: 'ElevatedButton', ); final rootNode = SemanticsNodeModel( @@ -295,7 +296,7 @@ void main() { final node = SemanticsNodeModel( id: '0', label: 'Checkbox Node', - flags: ['hasCheckedState'], + flags: {SemanticsFlag.hasCheckedState}, ); controller.semanticsRoots.value = [node]; From e099465c2a1d73d7185a602ac026e03fee400f98 Mon Sep 17 00:00:00 2001 From: hangyu Date: Mon, 31 Aug 2026 16:06:41 -0700 Subject: [PATCH 5/7] lint --- .../src/screens/accessibility/accessibility_controller.dart | 2 +- .../lib/src/screens/accessibility/semantics_node_model.dart | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index 6b910f85e89..8b449858323 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -249,7 +249,7 @@ class AccessibilityController extends DevToolsScreenController } SemanticsNodeModel _parseSemanticsNode(Map json) { - final rawFlags = json['flags'] as List?; + final rawFlags = json['flags'] as List?; final flags = SemanticsNodeModel.parseFlags(rawFlags); return SemanticsNodeModel( diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart index 12b356725be..2ee7209adc9 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_node_model.dart @@ -36,10 +36,10 @@ class SemanticsNodeModel extends TreeNode { }; /// Parses a list of flag name strings into a set of [SemanticsFlag]s. - static Set parseFlags(List? rawFlags) { + static Set parseFlags(List? rawFlags) { if (rawFlags == null) return const {}; return rawFlags - .map((e) => _flagByName[e.toString()]) + .map((e) => _flagByName[e?.toString()]) .whereType() .toSet(); } From 9eccc74b0d54c798433dad3ad549318d836645af Mon Sep 17 00:00:00 2001 From: hangyu Date: Tue, 1 Sep 2026 12:41:58 -0700 Subject: [PATCH 6/7] remove export --- .../lib/src/screens/accessibility/accessibility_controller.dart | 2 -- .../lib/src/screens/accessibility/semantics_tree_pane.dart | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index 8b449858323..0271fd5dac7 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -16,8 +16,6 @@ import '../../shared/framework/screen_controllers.dart'; import '../../shared/globals.dart'; import 'semantics_node_model.dart'; -export 'semantics_node_model.dart'; - final _log = Logger('accessibility_controller'); /// Modes for brightness override in the accessibility controls. diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart index cc8c3e0e16a..fb05497a124 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart @@ -12,6 +12,7 @@ import '../../shared/globals.dart'; import '../../shared/ui/common_widgets.dart'; import '../../shared/ui/tree_view.dart'; import 'accessibility_controller.dart'; +import 'semantics_node_model.dart'; /// A pane that displays the semantics tree of the connected app. class AccessibilitySemanticsTreePane extends StatelessWidget { From 58f8760c0071f815923414ab91cdad87747ad339 Mon Sep 17 00:00:00 2001 From: hangyu Date: Thu, 3 Sep 2026 13:54:50 -0700 Subject: [PATCH 7/7] resolve comments --- .../screens/accessibility/accessibility_controller.dart | 7 ++++++- .../lib/src/screens/accessibility/semantics_tree_pane.dart | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart index 0271fd5dac7..573756a67aa 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/accessibility_controller.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'package:devtools_app_shared/service.dart'; import 'package:devtools_app_shared/utils.dart'; import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart' show ScrollController; import 'package:logging/logging.dart'; import '../../service/service_extensions.dart' as extensions; @@ -149,6 +150,7 @@ class AccessibilityController extends DevToolsScreenController final semanticsRoots = ValueNotifier>([]); final semanticsTreeLoading = ValueNotifier(false); final semanticsTreeError = ValueNotifier(null); + final treeScrollController = ScrollController(); Future loadSemanticsTree() async { if (semanticsTreeLoading.value) return; @@ -209,7 +211,9 @@ class AccessibilityController extends DevToolsScreenController semanticsRoots.value = []; semanticsTreeError.value = 'Failed to load semantics tree: $e'; } finally { - semanticsTreeLoading.value = false; + if (!disposed) { + semanticsTreeLoading.value = false; + } } } @@ -269,6 +273,7 @@ class AccessibilityController extends DevToolsScreenController semanticsRoots.dispose(); semanticsTreeLoading.dispose(); semanticsTreeError.dispose(); + treeScrollController.dispose(); super.dispose(); } diff --git a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart index fb05497a124..9b9700742cb 100644 --- a/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart +++ b/packages/devtools_app/lib/src/screens/accessibility/semantics_tree_pane.dart @@ -169,6 +169,8 @@ class _SemanticsTreeContent extends StatelessWidget { return TreeView( dataRootsListenable: controller.semanticsRoots, + scrollController: controller.treeScrollController, + includeScrollbar: true, dataDisplayProvider: (node, onPressed) { return InkWell( onTap: onPressed,