Flutter Engine
accessibility_bridge.h
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
6 #define FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
7 
8 // Work around symbol conflicts with ICU.
9 #undef TRUE
10 #undef FALSE
11 
12 #include <fuchsia/accessibility/semantics/cpp/fidl.h>
13 #include <fuchsia/sys/cpp/fidl.h>
14 #include <fuchsia/ui/gfx/cpp/fidl.h>
15 #include <lib/fidl/cpp/binding_set.h>
16 #include <lib/sys/cpp/service_directory.h>
17 #include <zircon/types.h>
18 
19 #include <memory>
20 #include <optional>
21 #include <unordered_map>
22 #include <unordered_set>
23 #include <vector>
24 
25 #include "flutter/fml/macros.h"
26 #include "flutter/lib/ui/semantics/semantics_node.h"
27 
28 namespace flutter_runner {
29 // Accessibility bridge.
30 //
31 // This class intermediates accessibility-related calls between Fuchsia and
32 // Flutter. It serves to resolve the impedance mismatch between Flutter's
33 // platform-agnostic accessibility APIs and Fuchsia's APIs and behaviour.
34 //
35 // This bridge performs the following functions, among others:
36 //
37 // * Translates Flutter's semantics node updates to events Fuchsia requires
38 // (e.g. Flutter only sends updates for changed nodes, but Fuchsia requires
39 // the entire flattened subtree to be sent when a node changes.
41  : public fuchsia::accessibility::semantics::SemanticListener {
42  public:
43  // A delegate to call when semantics are enabled or disabled.
44  class Delegate {
45  public:
46  virtual void SetSemanticsEnabled(bool enabled) = 0;
47  virtual void DispatchSemanticsAction(int32_t node_id,
49  };
50 
51  // TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about
52  // batching messages at this level.
53  // FIDL may encode a C++ struct as larger than the sizeof the C++ struct.
54  // This is to make sure we don't send updates that are too large.
55  static constexpr uint32_t kMaxMessageSize = ZX_CHANNEL_MAX_MSG_BYTES / 2;
56 
57  static_assert(fuchsia::accessibility::semantics::MAX_LABEL_SIZE <
58  kMaxMessageSize - 1);
59 
60  // Flutter uses signed 32 bit integers for node IDs, while Fuchsia uses
61  // unsigned 32 bit integers. A change in the size on either one would break
62  // casts and size tracking logic in the implementation.
63  static constexpr size_t kNodeIdSize = sizeof(flutter::SemanticsNode::id);
64  static_assert(
65  kNodeIdSize ==
66  sizeof(fuchsia::accessibility::semantics::Node().node_id()),
67  "flutter::SemanticsNode::id and "
68  "fuchsia::accessibility::semantics::Node::node_id differ in size.");
69 
70  AccessibilityBridge(Delegate& delegate,
71  const std::shared_ptr<sys::ServiceDirectory> services,
72  fuchsia::ui::views::ViewRef view_ref);
73 
74  // Returns true if accessible navigation is enabled.
75  bool GetSemanticsEnabled() const;
76 
77  // Enables Flutter accessibility navigation features.
78  //
79  // Once enabled, any semantics updates in the Flutter application will
80  // trigger |FuchsiaAccessibility::DispatchAccessibilityEvent| callbacks
81  // to send events back to the Fuchsia SemanticsManager.
82  void SetSemanticsEnabled(bool enabled);
83 
84  // Adds a semantics node update to the buffer of node updates to apply.
86  float view_pixel_ratio);
87 
88  // Notifies the bridge of a 'hover move' touch exploration event.
89  zx_status_t OnHoverMove(double x, double y);
90 
91  // |fuchsia::accessibility::semantics::SemanticListener|
92  void HitTest(
93  fuchsia::math::PointF local_point,
94  fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
95  callback) override;
96 
97  // |fuchsia::accessibility::semantics::SemanticListener|
99  uint32_t node_id,
100  fuchsia::accessibility::semantics::Action action,
101  fuchsia::accessibility::semantics::SemanticListener::
102  OnAccessibilityActionRequestedCallback callback) override;
103 
104  private:
105  // Holds only the fields we need for hit testing.
106  // In particular, it adds a screen_rect field to flutter::SemanticsNode.
107  struct SemanticsNode {
108  int32_t id;
109  int32_t flags;
110  SkRect rect;
111  SkRect screen_rect;
112  SkM44 transform;
113  std::vector<int32_t> children_in_hit_test_order;
114  };
115 
117 
118  static constexpr int32_t kRootNodeId = 0;
119  flutter::SemanticsNode root_flutter_semantics_node_;
120  float last_seen_view_pixel_ratio_ = 1.f;
121  fidl::Binding<fuchsia::accessibility::semantics::SemanticListener> binding_;
122  fuchsia::accessibility::semantics::SemanticsManagerPtr
123  fuchsia_semantics_manager_;
124  fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
125  bool semantics_enabled_;
126  // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
127  // Assists with pruning unreachable nodes and hit testing.
128  std::unordered_map<int32_t, SemanticsNode> nodes_;
129 
130  fuchsia::accessibility::semantics::Node GetRootNodeUpdate(size_t& node_size);
131 
132  // Derives the BoundingBox of a Flutter semantics node from its
133  // rect and elevation.
134  fuchsia::ui::gfx::BoundingBox GetNodeLocation(
135  const flutter::SemanticsNode& node) const;
136 
137  // Gets mat4 transformation from a Flutter semantics node.
138  fuchsia::ui::gfx::mat4 GetNodeTransform(
139  const flutter::SemanticsNode& node) const;
140 
141  // Converts a Flutter semantics node's transformation to a mat4.
142  fuchsia::ui::gfx::mat4 ConvertSkiaTransformToMat4(
143  const SkM44 transform) const;
144 
145  // Derives the attributes for a Fuchsia semantics node from a Flutter
146  // semantics node.
147  fuchsia::accessibility::semantics::Attributes GetNodeAttributes(
148  const flutter::SemanticsNode& node,
149  size_t* added_size) const;
150 
151  // Derives the states for a Fuchsia semantics node from a Flutter semantics
152  // node.
153  fuchsia::accessibility::semantics::States GetNodeStates(
154  const flutter::SemanticsNode& node,
155  size_t* additional_size) const;
156 
157  // Derives the set of supported actions for a Fuchsia semantics node from
158  // a Flutter semantics node.
159  std::vector<fuchsia::accessibility::semantics::Action> GetNodeActions(
160  const flutter::SemanticsNode& node,
161  size_t* additional_size) const;
162 
163  // Derives the role for a Fuchsia semantics node from a Flutter
164  // semantics node.
165  fuchsia::accessibility::semantics::Role GetNodeRole(
166  const flutter::SemanticsNode& node) const;
167 
168  // Gets the set of reachable descendants from the given node id.
169  std::unordered_set<int32_t> GetDescendants(int32_t node_id) const;
170 
171  // Removes internal references to any dangling nodes from previous
172  // updates, and updates the Accessibility service.
173  //
174  // May result in a call to FuchsiaAccessibility::Commit().
175  void PruneUnreachableNodes();
176 
177  // Updates the on-screen positions of accessibility elements,
178  // starting from the root element with an identity matrix.
179  //
180  // This should be called from Update.
181  void UpdateScreenRects();
182 
183  // Updates the on-screen positions of accessibility elements, starting
184  // from node_id and using the specified transform.
185  //
186  // Update calls this via UpdateScreenRects().
187  void UpdateScreenRects(int32_t node_id,
188  SkM44 parent_transform,
189  std::unordered_set<int32_t>* visited_nodes);
190 
191  // Traverses the semantics tree to find the node_id hit by the given x,y
192  // point.
193  //
194  // Assumes that SemanticsNode::screen_rect is up to date.
195  std::optional<int32_t> GetHitNode(int32_t node_id, float x, float y);
196 
197  // Converts a fuchsia::accessibility::semantics::Action to a
198  // flutter::SemanticsAction.
199  //
200  // The node_id parameter is used for printing warnings about unsupported
201  // action types.
202  std::optional<flutter::SemanticsAction> GetFlutterSemanticsAction(
203  fuchsia::accessibility::semantics::Action fuchsia_action,
204  uint32_t node_id);
205 
206  // |fuchsia::accessibility::semantics::SemanticListener|
207  void OnSemanticsModeChanged(bool enabled,
208  OnSemanticsModeChangedCallback callback) override;
209 
211 };
212 } // namespace flutter_runner
213 
214 #endif // FLUTTER_SHELL_PLATFORM_FUCHSIA_FLUTTER_ACCESSIBILITY_BRIDGE_H_
virtual void DispatchSemanticsAction(int32_t node_id, flutter::SemanticsAction action)=0
void OnAccessibilityActionRequested(uint32_t node_id, fuchsia::accessibility::semantics::Action action, fuchsia::accessibility::semantics::SemanticListener::OnAccessibilityActionRequestedCallback callback) override
std::unordered_map< int32_t, SemanticsNode > SemanticsNodeUpdates
MockDelegate delegate_
zx_status_t OnHoverMove(double x, double y)
virtual void SetSemanticsEnabled(bool enabled)=0
SemanticsAction action
AccessibilityBridge(Delegate &delegate, const std::shared_ptr< sys::ServiceDirectory > services, fuchsia::ui::views::ViewRef view_ref)
static constexpr uint32_t kMaxMessageSize
constexpr int32_t kRootNodeId
int32_t id
void HitTest(fuchsia::math::PointF local_point, fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) override
void AddSemanticsNodeUpdate(const flutter::SemanticsNodeUpdates update, float view_pixel_ratio)
#define FML_DISALLOW_COPY_AND_ASSIGN(TypeName)
Definition: macros.h:27
DEF_SWITCHES_START snapshot asset Path to the directory containing the four files specified by VmSnapshotInstructions and IsolateSnapshotInstructions vm snapshot The VM instructions snapshot that will be memory mapped as read and executable SnapshotAssetPath must be present isolate snapshot The isolate instructions snapshot that will be memory mapped as read and executable SnapshotAssetPath must be present icu symbol Prefix for the symbols representing ICU data linked into the Flutter library dart flags
Definition: switches.h:66