Flutter Engine
accessibility_bridge.h
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
6 #define FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
7 
8 // Work around symbol conflicts with ICU.
9 #undef TRUE
10 #undef FALSE
11 
12 #include <fuchsia/accessibility/semantics/cpp/fidl.h>
13 #include <fuchsia/sys/cpp/fidl.h>
14 #include <fuchsia/ui/gfx/cpp/fidl.h>
15 #include <lib/fidl/cpp/binding_set.h>
16 #include <lib/sys/inspect/cpp/component.h>
17 #include <zircon/types.h>
18 
19 #include <memory>
20 #include <optional>
21 #include <unordered_map>
22 #include <unordered_set>
23 #include <vector>
24 
25 #include "flutter/fml/macros.h"
26 #include "flutter/lib/ui/semantics/semantics_node.h"
27 
28 namespace flutter_runner {
29 // Accessibility bridge.
30 //
31 // This class intermediates accessibility-related calls between Fuchsia and
32 // Flutter. It serves to resolve the impedance mismatch between Flutter's
33 // platform-agnostic accessibility APIs and Fuchsia's APIs and behaviour.
34 //
35 // This bridge performs the following functions, among others:
36 //
37 // * Translates Flutter's semantics node updates to events Fuchsia requires
38 // (e.g. Flutter only sends updates for changed nodes, but Fuchsia requires
39 // the entire flattened subtree to be sent when a node changes.
41  : public fuchsia::accessibility::semantics::SemanticListener {
42  public:
43  using SetSemanticsEnabledCallback = std::function<void(bool)>;
45  std::function<void(int32_t, flutter::SemanticsAction)>;
46 
47  // TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about
48  // batching messages at this level.
49  // FIDL may encode a C++ struct as larger than the sizeof the C++ struct.
50  // This is to make sure we don't send updates that are too large.
51  static constexpr uint32_t kMaxMessageSize = ZX_CHANNEL_MAX_MSG_BYTES / 2;
52 
53  static_assert(fuchsia::accessibility::semantics::MAX_LABEL_SIZE <
54  kMaxMessageSize - 1);
55 
56  // Flutter uses signed 32 bit integers for node IDs, while Fuchsia uses
57  // unsigned 32 bit integers. A change in the size on either one would break
58  // casts and size tracking logic in the implementation.
59  static constexpr size_t kNodeIdSize = sizeof(flutter::SemanticsNode::id);
60  static_assert(
61  kNodeIdSize ==
62  sizeof(fuchsia::accessibility::semantics::Node().node_id()),
63  "flutter::SemanticsNode::id and "
64  "fuchsia::accessibility::semantics::Node::node_id differ in size.");
65 
67  SetSemanticsEnabledCallback set_semantics_enabled_callback,
68  DispatchSemanticsActionCallback dispatch_semantics_action_callback,
69  fuchsia::accessibility::semantics::SemanticsManagerHandle
70  semantics_manager,
71  fuchsia::ui::views::ViewRef view_ref,
72  inspect::Node inspect_node);
73 
74  // Returns true if accessible navigation is enabled.
75  bool GetSemanticsEnabled() const;
76 
77  // Enables Flutter accessibility navigation features.
78  //
79  // Once enabled, any semantics updates in the Flutter application will
80  // trigger |FuchsiaAccessibility::DispatchAccessibilityEvent| callbacks
81  // to send events back to the Fuchsia SemanticsManager.
82  void SetSemanticsEnabled(bool enabled);
83 
84  // Adds a semantics node update to the buffer of node updates to apply.
86  float view_pixel_ratio);
87 
88  // Requests a message announcement from the accessibility TTS system.
89  void RequestAnnounce(const std::string message);
90 
91  // Notifies the bridge of a 'hover move' touch exploration event.
92  zx_status_t OnHoverMove(double x, double y);
93 
94  // |fuchsia::accessibility::semantics::SemanticListener|
95  void HitTest(
96  fuchsia::math::PointF local_point,
97  fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
98  callback) override;
99 
100  // |fuchsia::accessibility::semantics::SemanticListener|
102  uint32_t node_id,
104  fuchsia::accessibility::semantics::SemanticListener::
105  OnAccessibilityActionRequestedCallback callback) override;
106 
107  private:
108  // Fuchsia's default root semantic node id.
109  static constexpr int32_t kRootNodeId = 0;
110 
111  // Holds a flutter semantic node and some extra info.
112  // In particular, it adds a screen_rect field to flutter::SemanticsNode.
113  struct SemanticsNode {
115  SkRect screen_rect;
116  };
117 
118  fuchsia::accessibility::semantics::Node GetRootNodeUpdate(size_t& node_size);
119 
120  // Derives the BoundingBox of a Flutter semantics node from its
121  // rect and elevation.
122  fuchsia::ui::gfx::BoundingBox GetNodeLocation(
123  const flutter::SemanticsNode& node) const;
124 
125  // Gets mat4 transformation from a Flutter semantics node.
126  fuchsia::ui::gfx::mat4 GetNodeTransform(
127  const flutter::SemanticsNode& node) const;
128 
129  // Converts a Flutter semantics node's transformation to a mat4.
130  fuchsia::ui::gfx::mat4 ConvertSkiaTransformToMat4(
131  const SkM44 transform) const;
132 
133  // Derives the attributes for a Fuchsia semantics node from a Flutter
134  // semantics node.
135  fuchsia::accessibility::semantics::Attributes GetNodeAttributes(
137  size_t* added_size) const;
138 
139  // Derives the states for a Fuchsia semantics node from a Flutter semantics
140  // node.
141  fuchsia::accessibility::semantics::States GetNodeStates(
143  size_t* additional_size) const;
144 
145  // Derives the set of supported actions for a Fuchsia semantics node from
146  // a Flutter semantics node.
147  std::vector<fuchsia::accessibility::semantics::Action> GetNodeActions(
149  size_t* additional_size) const;
150 
151  // Derives the role for a Fuchsia semantics node from a Flutter
152  // semantics node.
154  const flutter::SemanticsNode& node) const;
155 
156  // Gets the set of reachable descendants from the given node id.
157  std::unordered_set<int32_t> GetDescendants(int32_t node_id) const;
158 
159  // Removes internal references to any dangling nodes from previous
160  // updates, and updates the Accessibility service.
161  //
162  // May result in a call to FuchsiaAccessibility::Commit().
163  void PruneUnreachableNodes();
164 
165  // Updates the on-screen positions of accessibility elements,
166  // starting from the root element with an identity matrix.
167  //
168  // This should be called from Update.
169  void UpdateScreenRects();
170 
171  // Updates the on-screen positions of accessibility elements, starting
172  // from node_id and using the specified transform.
173  //
174  // Update calls this via UpdateScreenRects().
175  void UpdateScreenRects(int32_t node_id,
176  SkM44 parent_transform,
177  std::unordered_set<int32_t>* visited_nodes);
178 
179  // Traverses the semantics tree to find the node_id hit by the given x,y
180  // point.
181  //
182  // Assumes that SemanticsNode::screen_rect is up to date.
183  std::optional<int32_t> GetHitNode(int32_t node_id, float x, float y);
184 
185  // Returns whether the node is considered focusable.
186  bool IsFocusable(const flutter::SemanticsNode& node) const;
187 
188  // Converts a fuchsia::accessibility::semantics::Action to a
189  // flutter::SemanticsAction.
190  //
191  // The node_id parameter is used for printing warnings about unsupported
192  // action types.
193  std::optional<flutter::SemanticsAction> GetFlutterSemanticsAction(
195  uint32_t node_id);
196 
197  // |fuchsia::accessibility::semantics::SemanticListener|
198  void OnSemanticsModeChanged(bool enabled,
199  OnSemanticsModeChangedCallback callback) override;
200 
201 #if !FLUTTER_RELEASE
202  // Fills the inspect tree with debug information about the semantic tree.
203  void FillInspectTree(int32_t flutter_node_id,
204  int32_t current_level,
205  inspect::Node inspect_node,
206  inspect::Inspector* inspector) const;
207 #endif // !FLUTTER_RELEASE
208 
209  SetSemanticsEnabledCallback set_semantics_enabled_callback_;
210  DispatchSemanticsActionCallback dispatch_semantics_action_callback_;
211  flutter::SemanticsNode root_flutter_semantics_node_;
212  float last_seen_view_pixel_ratio_ = 1.f;
213 
214  fidl::Binding<fuchsia::accessibility::semantics::SemanticListener> binding_;
215  fuchsia::accessibility::semantics::SemanticsManagerPtr
216  fuchsia_semantics_manager_;
217  fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
218 
219  // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
220  // Assists with pruning unreachable nodes and hit testing.
221  std::unordered_map<int32_t, SemanticsNode> nodes_;
222  bool semantics_enabled_;
223 
224  // Node to publish inspect data.
225  inspect::Node inspect_node_;
226 
227 #if !FLUTTER_RELEASE
228  // Inspect node to store a dump of the semantic tree. Note that this only gets
229  // computed if requested, so it does not use memory to store the dump unless
230  // an explicit request is made.
231  inspect::LazyNode inspect_node_tree_dump_;
232 #endif // !FLUTTER_RELEASE
233 
235 };
236 } // namespace flutter_runner
237 
238 #endif // FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
void RequestAnnounce(const std::string message)
void OnAccessibilityActionRequested(uint32_t node_id, fuchsia::accessibility::semantics::Action action, fuchsia::accessibility::semantics::SemanticListener::OnAccessibilityActionRequestedCallback callback) override
std::unordered_map< int32_t, SemanticsNode > SemanticsNodeUpdates
FlKeyEvent FlKeyResponderAsyncCallback callback
zx_status_t OnHoverMove(double x, double y)
DEF_SWITCHES_START aot vmservice shared library Name of the *so containing AOT compiled Dart assets for launching the service isolate vm snapshot data
Definition: switches.h:41
const FlutterSemanticsNode * node
Definition: fl_view.cc:83
SemanticsAction action
AccessibilityBridge(SetSemanticsEnabledCallback set_semantics_enabled_callback, DispatchSemanticsActionCallback dispatch_semantics_action_callback, fuchsia::accessibility::semantics::SemanticsManagerHandle semantics_manager, fuchsia::ui::views::ViewRef view_ref, inspect::Node inspect_node)
std::function< void(bool)> SetSemanticsEnabledCallback
static constexpr uint32_t kMaxMessageSize
void HitTest(fuchsia::math::PointF local_point, fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) override
void AddSemanticsNodeUpdate(const flutter::SemanticsNodeUpdates update, float view_pixel_ratio)
#define FML_DISALLOW_COPY_AND_ASSIGN(TypeName)
Definition: macros.h:27
std::function< void(int32_t, flutter::SemanticsAction)> DispatchSemanticsActionCallback