Flutter Engine
accessibility_bridge.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h"
6 
7 #include <zircon/status.h>
8 #include <zircon/types.h>
9 
10 #include <deque>
11 
12 #include "flutter/fml/logging.h"
13 #include "flutter/lib/ui/semantics/semantics_node.h"
14 
15 namespace flutter_runner {
17  Delegate& delegate,
18  const std::shared_ptr<sys::ServiceDirectory> services,
19  fuchsia::ui::views::ViewRef view_ref)
20  : delegate_(delegate), binding_(this) {
21  services->Connect(fuchsia::accessibility::semantics::SemanticsManager::Name_,
22  fuchsia_semantics_manager_.NewRequest().TakeChannel());
23  fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) {
24  FML_LOG(ERROR) << "Flutter cannot connect to SemanticsManager with status: "
25  << zx_status_get_string(status) << ".";
26  });
27  fidl::InterfaceHandle<fuchsia::accessibility::semantics::SemanticListener>
28  listener_handle;
29  binding_.Bind(listener_handle.NewRequest());
30  fuchsia_semantics_manager_->RegisterViewForSemantics(
31  std::move(view_ref), std::move(listener_handle), tree_ptr_.NewRequest());
32 }
33 
35  return semantics_enabled_;
36 }
37 
39  semantics_enabled_ = enabled;
40  if (!enabled) {
41  nodes_.clear();
42  }
43 }
44 
45 fuchsia::ui::gfx::BoundingBox AccessibilityBridge::GetNodeLocation(
46  const flutter::SemanticsNode& node) const {
47  fuchsia::ui::gfx::BoundingBox box;
48  box.min.x = node.rect.fLeft;
49  box.min.y = node.rect.fTop;
50  box.min.z = static_cast<float>(node.elevation);
51  box.max.x = node.rect.fRight;
52  box.max.y = node.rect.fBottom;
53  box.max.z = static_cast<float>(node.thickness);
54  return box;
55 }
56 
57 fuchsia::ui::gfx::mat4 AccessibilityBridge::GetNodeTransform(
58  const flutter::SemanticsNode& node) const {
59  return ConvertSkiaTransformToMat4(node.transform);
60 }
61 
62 fuchsia::ui::gfx::mat4 AccessibilityBridge::ConvertSkiaTransformToMat4(
63  const SkM44 transform) const {
64  fuchsia::ui::gfx::mat4 value;
65  float* m = value.matrix.data();
66  transform.getColMajor(m);
67  return value;
68 }
69 
70 fuchsia::accessibility::semantics::Attributes
71 AccessibilityBridge::GetNodeAttributes(const flutter::SemanticsNode& node,
72  size_t* added_size) const {
73  fuchsia::accessibility::semantics::Attributes attributes;
74  // TODO(MI4-2531): Don't truncate.
75  if (node.label.size() > fuchsia::accessibility::semantics::MAX_LABEL_SIZE) {
76  attributes.set_label(node.label.substr(
77  0, fuchsia::accessibility::semantics::MAX_LABEL_SIZE));
78  *added_size += fuchsia::accessibility::semantics::MAX_LABEL_SIZE;
79  } else {
80  attributes.set_label(node.label);
81  *added_size += node.label.size();
82  }
83 
84  return attributes;
85 }
86 
87 fuchsia::accessibility::semantics::States AccessibilityBridge::GetNodeStates(
88  const flutter::SemanticsNode& node,
89  size_t* additional_size) const {
90  fuchsia::accessibility::semantics::States states;
91  (*additional_size) += sizeof(fuchsia::accessibility::semantics::States);
92 
93  // Set checked state.
95  states.set_checked_state(
96  fuchsia::accessibility::semantics::CheckedState::NONE);
97  } else {
98  states.set_checked_state(
100  ? fuchsia::accessibility::semantics::CheckedState::CHECKED
101  : fuchsia::accessibility::semantics::CheckedState::UNCHECKED);
102  }
103 
104  // Set selected state.
105  states.set_selected(node.HasFlag(flutter::SemanticsFlags::kIsSelected));
106 
107  // Flutter's definition of a hidden node is different from Fuchsia, so it must
108  // not be set here.
109 
110  // Set value.
111  if (node.value.size() > fuchsia::accessibility::semantics::MAX_VALUE_SIZE) {
112  states.set_value(node.value.substr(
113  0, fuchsia::accessibility::semantics::MAX_VALUE_SIZE));
114  (*additional_size) += fuchsia::accessibility::semantics::MAX_VALUE_SIZE;
115  } else {
116  states.set_value(node.value);
117  (*additional_size) += node.value.size();
118  }
119 
120  return states;
121 }
122 
123 std::vector<fuchsia::accessibility::semantics::Action>
124 AccessibilityBridge::GetNodeActions(const flutter::SemanticsNode& node,
125  size_t* additional_size) const {
126  std::vector<fuchsia::accessibility::semantics::Action> node_actions;
127 
129  node_actions.push_back(fuchsia::accessibility::semantics::Action::DEFAULT);
130  }
132  node_actions.push_back(
133  fuchsia::accessibility::semantics::Action::SECONDARY);
134  }
136  node_actions.push_back(
137  fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN);
138  }
140  node_actions.push_back(
141  fuchsia::accessibility::semantics::Action::INCREMENT);
142  }
144  node_actions.push_back(
145  fuchsia::accessibility::semantics::Action::DECREMENT);
146  }
147 
148  *additional_size +=
149  node_actions.size() * sizeof(fuchsia::accessibility::semantics::Action);
150  return node_actions;
151 }
152 
153 fuchsia::accessibility::semantics::Role AccessibilityBridge::GetNodeRole(
154  const flutter::SemanticsNode& node) const {
156  return fuchsia::accessibility::semantics::Role::BUTTON;
157  }
158 
160  return fuchsia::accessibility::semantics::Role::TEXT_FIELD;
161  }
162 
164  return fuchsia::accessibility::semantics::Role::LINK;
165  }
166 
168  return fuchsia::accessibility::semantics::Role::SLIDER;
169  }
170 
172  return fuchsia::accessibility::semantics::Role::HEADER;
173  }
175  return fuchsia::accessibility::semantics::Role::IMAGE;
176  }
177 
178  // If a flutter node supports the kIncrease or kDecrease actions, it can be
179  // treated as a slider control by assistive technology. This is important
180  // because users have special gestures to deal with sliders, and Fuchsia API
181  // requires nodes that can receive this kind of action to be a slider control.
184  return fuchsia::accessibility::semantics::Role::SLIDER;
185  }
186 
187  // If a flutter node has a checked state, then we assume it is either a
188  // checkbox or a radio button. We distinguish between checkboxes and
189  // radio buttons based on membership in a mutually exclusive group.
192  return fuchsia::accessibility::semantics::Role::RADIO_BUTTON;
193  } else {
194  return fuchsia::accessibility::semantics::Role::CHECK_BOX;
195  }
196  }
197 
198  return fuchsia::accessibility::semantics::Role::UNKNOWN;
199 }
200 
201 std::unordered_set<int32_t> AccessibilityBridge::GetDescendants(
202  int32_t node_id) const {
203  std::unordered_set<int32_t> descendents;
204  std::deque<int32_t> to_process = {node_id};
205  while (!to_process.empty()) {
206  int32_t id = to_process.front();
207  to_process.pop_front();
208  descendents.emplace(id);
209 
210  auto it = nodes_.find(id);
211  if (it != nodes_.end()) {
212  const auto& node = it->second;
213  for (const auto& child : node.children_in_hit_test_order) {
214  if (descendents.find(child) == descendents.end()) {
215  to_process.push_back(child);
216  } else {
217  // This indicates either a cycle or a child with multiple parents.
218  // Flutter should never let this happen, but the engine API does not
219  // explicitly forbid it right now.
220  FML_LOG(ERROR) << "Semantics Node " << child
221  << " has already been listed as a child of another "
222  "node, ignoring for parent "
223  << id << ".";
224  }
225  }
226  }
227  }
228  return descendents;
229 }
230 
231 // The only known usage of a negative number for a node ID is in the embedder
232 // API as a sentinel value, which is not expected here. No valid producer of
233 // nodes should give us a negative ID.
234 static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id) {
235  FML_DCHECK(flutter_node_id >= 0)
236  << "Unexpectedly recieved a negative semantics node ID.";
237  return static_cast<uint32_t>(flutter_node_id);
238 }
239 
240 void AccessibilityBridge::PruneUnreachableNodes() {
241  const auto& reachable_nodes = GetDescendants(kRootNodeId);
242  std::vector<uint32_t> nodes_to_remove;
243  auto iter = nodes_.begin();
244  while (iter != nodes_.end()) {
245  int32_t id = iter->first;
246  if (reachable_nodes.find(id) == reachable_nodes.end()) {
247  // TODO(MI4-2531): This shouldn't be strictly necessary at this level.
248  if (sizeof(nodes_to_remove) + (nodes_to_remove.size() * kNodeIdSize) >=
249  kMaxMessageSize) {
250  tree_ptr_->DeleteSemanticNodes(std::move(nodes_to_remove));
251  nodes_to_remove.clear();
252  }
253  nodes_to_remove.push_back(FlutterIdToFuchsiaId(id));
254  iter = nodes_.erase(iter);
255  } else {
256  iter++;
257  }
258  }
259  if (!nodes_to_remove.empty()) {
260  tree_ptr_->DeleteSemanticNodes(std::move(nodes_to_remove));
261  }
262 }
263 
264 // TODO(FIDL-718) - remove this, handle the error instead in something like
265 // set_error_handler.
266 static void PrintNodeSizeError(uint32_t node_id) {
267  FML_LOG(ERROR) << "Semantics node with ID " << node_id
268  << " exceeded the maximum FIDL message size and may not "
269  "be delivered to the accessibility manager service.";
270 }
271 
273  const flutter::SemanticsNodeUpdates update,
274  float view_pixel_ratio) {
275  if (update.empty()) {
276  return;
277  }
278  FML_DCHECK(nodes_.find(kRootNodeId) != nodes_.end() ||
279  update.find(kRootNodeId) != update.end())
280  << "AccessibilityBridge received an update with out ever getting a root "
281  "node.";
282 
283  std::vector<fuchsia::accessibility::semantics::Node> nodes;
284  size_t current_size = 0;
285  bool has_root_node_update = false;
286  // TODO(MI4-2498): Actions, Roles, hit test children, additional
287  // flags/states/attr
288 
289  // TODO(MI4-1478): Support for partial updates for nodes > 64kb
290  // e.g. if a node has a long label or more than 64k children.
291  for (const auto& value : update) {
292  size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node);
293  const auto& flutter_node = value.second;
294  // We handle root update separately in GetRootNodeUpdate.
295  // TODO(chunhtai): remove this special case after we remove the inverse
296  // view pixel ratio transformation in scenic view.
297  if (flutter_node.id == kRootNodeId) {
298  root_flutter_semantics_node_ = flutter_node;
299  has_root_node_update = true;
300  continue;
301  }
302  // Store the nodes for later hit testing.
303  nodes_[flutter_node.id] = {
304  .id = flutter_node.id,
305  .flags = flutter_node.flags,
306  .rect = flutter_node.rect,
307  .transform = flutter_node.transform,
308  .children_in_hit_test_order = flutter_node.childrenInHitTestOrder,
309  };
310  fuchsia::accessibility::semantics::Node fuchsia_node;
311  std::vector<uint32_t> child_ids;
312  // Send the nodes in traversal order, so the manager can figure out
313  // traversal.
314  for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) {
315  child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id));
316  }
317  fuchsia_node.set_node_id(flutter_node.id)
318  .set_role(GetNodeRole(flutter_node))
319  .set_location(GetNodeLocation(flutter_node))
320  .set_transform(GetNodeTransform(flutter_node))
321  .set_attributes(GetNodeAttributes(flutter_node, &this_node_size))
322  .set_states(GetNodeStates(flutter_node, &this_node_size))
323  .set_actions(GetNodeActions(flutter_node, &this_node_size))
324  .set_role(GetNodeRole(flutter_node))
325  .set_child_ids(child_ids);
326  this_node_size +=
327  kNodeIdSize * flutter_node.childrenInTraversalOrder.size();
328 
329  // TODO(MI4-2531, FIDL-718): Remove this
330  // This is defensive. If, despite our best efforts, we ended up with a node
331  // that is larger than the max fidl size, we send no updates.
332  if (this_node_size >= kMaxMessageSize) {
333  PrintNodeSizeError(flutter_node.id);
334  return;
335  }
336  current_size += this_node_size;
337 
338  // If we would exceed the max FIDL message size by appending this node,
339  // we should delete/update/commit now.
340  if (current_size >= kMaxMessageSize) {
341  tree_ptr_->UpdateSemanticNodes(std::move(nodes));
342  nodes.clear();
343  current_size = this_node_size;
344  }
345  nodes.push_back(std::move(fuchsia_node));
346  }
347 
348  if (current_size > kMaxMessageSize) {
349  PrintNodeSizeError(nodes.back().node_id());
350  }
351 
352  // Handles root node update.
353  if (has_root_node_update || last_seen_view_pixel_ratio_ != view_pixel_ratio) {
354  last_seen_view_pixel_ratio_ = view_pixel_ratio;
355  size_t root_node_size;
356  fuchsia::accessibility::semantics::Node root_update =
357  GetRootNodeUpdate(root_node_size);
358  // TODO(MI4-2531, FIDL-718): Remove this
359  // This is defensive. If, despite our best efforts, we ended up with a node
360  // that is larger than the max fidl size, we send no updates.
361  if (root_node_size >= kMaxMessageSize) {
362  PrintNodeSizeError(kRootNodeId);
363  return;
364  }
365  current_size += root_node_size;
366  // If we would exceed the max FIDL message size by appending this node,
367  // we should delete/update/commit now.
368  if (current_size >= kMaxMessageSize) {
369  tree_ptr_->UpdateSemanticNodes(std::move(nodes));
370  nodes.clear();
371  }
372  nodes.push_back(std::move(root_update));
373  }
374 
375  PruneUnreachableNodes();
376  UpdateScreenRects();
377 
378  tree_ptr_->UpdateSemanticNodes(std::move(nodes));
379  // TODO(dnfield): Implement the callback here
380  // https://bugs.fuchsia.dev/p/fuchsia/issues/detail?id=35718.
381  tree_ptr_->CommitUpdates([]() {});
382 }
383 
384 fuchsia::accessibility::semantics::Node AccessibilityBridge::GetRootNodeUpdate(
385  size_t& node_size) {
386  fuchsia::accessibility::semantics::Node root_fuchsia_node;
387  std::vector<uint32_t> child_ids;
388  node_size = sizeof(fuchsia::accessibility::semantics::Node);
389  for (int32_t flutter_child_id :
390  root_flutter_semantics_node_.childrenInTraversalOrder) {
391  child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id));
392  }
393  // Applies the inverse view pixel ratio transformation to the root node.
394  float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_;
395  SkM44 inverse_view_pixel_ratio_transform;
396  inverse_view_pixel_ratio_transform.setScale(inverse_view_pixel_ratio,
397  inverse_view_pixel_ratio, 1.f);
398 
399  SkM44 result = root_flutter_semantics_node_.transform *
400  inverse_view_pixel_ratio_transform;
401  nodes_[root_flutter_semantics_node_.id] = {
402  .id = root_flutter_semantics_node_.id,
403  .flags = root_flutter_semantics_node_.flags,
404  .rect = root_flutter_semantics_node_.rect,
405  .transform = result,
406  .children_in_hit_test_order =
407  root_flutter_semantics_node_.childrenInHitTestOrder,
408  };
409  root_fuchsia_node.set_node_id(root_flutter_semantics_node_.id)
410  .set_role(GetNodeRole(root_flutter_semantics_node_))
411  .set_location(GetNodeLocation(root_flutter_semantics_node_))
412  .set_transform(ConvertSkiaTransformToMat4(result))
413  .set_attributes(
414  GetNodeAttributes(root_flutter_semantics_node_, &node_size))
415  .set_states(GetNodeStates(root_flutter_semantics_node_, &node_size))
416  .set_actions(GetNodeActions(root_flutter_semantics_node_, &node_size))
417  .set_child_ids(child_ids);
418  node_size += kNodeIdSize *
419  root_flutter_semantics_node_.childrenInTraversalOrder.size();
420  return root_fuchsia_node;
421 }
422 
423 void AccessibilityBridge::UpdateScreenRects() {
424  std::unordered_set<int32_t> visited_nodes;
425  UpdateScreenRects(kRootNodeId, SkM44{}, &visited_nodes);
426 }
427 
428 void AccessibilityBridge::UpdateScreenRects(
429  int32_t node_id,
430  SkM44 parent_transform,
431  std::unordered_set<int32_t>* visited_nodes) {
432  auto it = nodes_.find(node_id);
433  if (it == nodes_.end()) {
434  FML_LOG(ERROR) << "UpdateScreenRects called on unknown node";
435  return;
436  }
437  auto& node = it->second;
438  const auto& current_transform = parent_transform * node.transform;
439 
440  const auto& rect = node.rect;
441  SkV4 dst[2] = {
442  current_transform.map(rect.left(), rect.top(), 0, 1),
443  current_transform.map(rect.right(), rect.bottom(), 0, 1),
444  };
445  node.screen_rect.setLTRB(dst[0].x, dst[0].y, dst[1].x, dst[1].y);
446  node.screen_rect.sort();
447 
448  visited_nodes->emplace(node_id);
449 
450  for (uint32_t child_id : node.children_in_hit_test_order) {
451  if (visited_nodes->find(child_id) == visited_nodes->end()) {
452  UpdateScreenRects(child_id, current_transform, visited_nodes);
453  }
454  }
455 }
456 
457 std::optional<flutter::SemanticsAction>
458 AccessibilityBridge::GetFlutterSemanticsAction(
459  fuchsia::accessibility::semantics::Action fuchsia_action,
460  uint32_t node_id) {
461  switch (fuchsia_action) {
462  // The default action associated with the element.
463  case fuchsia::accessibility::semantics::Action::DEFAULT:
465  // The secondary action associated with the element. This may correspond to
466  // a long press (touchscreens) or right click (mouse).
467  case fuchsia::accessibility::semantics::Action::SECONDARY:
469  // Set (input/non-accessibility) focus on this element.
470  case fuchsia::accessibility::semantics::Action::SET_FOCUS:
471  FML_DLOG(WARNING)
472  << "Unsupported action SET_FOCUS sent for accessibility node "
473  << node_id;
474  return {};
475  // Set the element's value.
476  case fuchsia::accessibility::semantics::Action::SET_VALUE:
477  FML_DLOG(WARNING)
478  << "Unsupported action SET_VALUE sent for accessibility node "
479  << node_id;
480  return {};
481  // Scroll node to make it visible.
482  case fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN:
484  case fuchsia::accessibility::semantics::Action::INCREMENT:
486  case fuchsia::accessibility::semantics::Action::DECREMENT:
488  default:
489  FML_DLOG(WARNING) << "Unexpected action "
490  << static_cast<int32_t>(fuchsia_action)
491  << " sent for accessibility node " << node_id;
492  return {};
493  }
494 }
495 
496 // |fuchsia::accessibility::semantics::SemanticListener|
498  uint32_t node_id,
499  fuchsia::accessibility::semantics::Action action,
500  fuchsia::accessibility::semantics::SemanticListener::
501  OnAccessibilityActionRequestedCallback callback) {
502  if (nodes_.find(node_id) == nodes_.end()) {
503  FML_LOG(ERROR) << "Attempted to send accessibility action "
504  << static_cast<int32_t>(action)
505  << " to unkonwn node id: " << node_id;
506  callback(false);
507  return;
508  }
509 
510  std::optional<flutter::SemanticsAction> flutter_action =
511  GetFlutterSemanticsAction(action, node_id);
512  if (!flutter_action.has_value()) {
513  callback(false);
514  return;
515  }
516  delegate_.DispatchSemanticsAction(static_cast<int32_t>(node_id),
517  flutter_action.value());
518  callback(true);
519 }
520 
521 // |fuchsia::accessibility::semantics::SemanticListener|
523  fuchsia::math::PointF local_point,
524  fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
525  callback) {
526  auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y);
527  FML_DCHECK(hit_node_id.has_value());
528  fuchsia::accessibility::semantics::Hit hit;
529  hit.set_node_id(hit_node_id.value_or(kRootNodeId));
530  callback(std::move(hit));
531 }
532 
533 std::optional<int32_t> AccessibilityBridge::GetHitNode(int32_t node_id,
534  float x,
535  float y) {
536  auto it = nodes_.find(node_id);
537  if (it == nodes_.end()) {
538  FML_LOG(ERROR) << "Attempted to hit test unkonwn node id: " << node_id;
539  return {};
540  }
541  auto const& node = it->second;
542  if (node.flags &
543  static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden) || //
544  !node.screen_rect.contains(x, y)) {
545  return {};
546  }
547  for (int32_t child_id : node.children_in_hit_test_order) {
548  auto candidate = GetHitNode(child_id, x, y);
549  if (candidate) {
550  return candidate;
551  }
552  }
553  return node_id;
554 }
555 
556 // |fuchsia::accessibility::semantics::SemanticListener|
557 void AccessibilityBridge::OnSemanticsModeChanged(
558  bool enabled,
559  OnSemanticsModeChangedCallback callback) {
560  delegate_.SetSemanticsEnabled(enabled);
561 }
562 
563 } // namespace flutter_runner
std::vector< int32_t > childrenInHitTestOrder
#define FML_DCHECK(condition)
Definition: logging.h:86
virtual void DispatchSemanticsAction(int32_t node_id, flutter::SemanticsAction action)=0
void OnAccessibilityActionRequested(uint32_t node_id, fuchsia::accessibility::semantics::Action action, fuchsia::accessibility::semantics::SemanticListener::OnAccessibilityActionRequestedCallback callback) override
std::unordered_map< int32_t, SemanticsNode > SemanticsNodeUpdates
#define FML_LOG(severity)
Definition: logging.h:65
std::vector< int32_t > childrenInTraversalOrder
bool HasFlag(SemanticsFlags flag) const
MockDelegate delegate_
virtual void SetSemanticsEnabled(bool enabled)=0
bool HasAction(SemanticsAction action) const
uint8_t value
static void PrintNodeSizeError(uint32_t node_id)
SemanticsAction action
AccessibilityBridge(Delegate &delegate, const std::shared_ptr< sys::ServiceDirectory > services, fuchsia::ui::views::ViewRef view_ref)
static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id)
static constexpr uint32_t kMaxMessageSize
#define FML_DLOG(severity)
Definition: logging.h:85
void HitTest(fuchsia::math::PointF local_point, fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) override
void AddSemanticsNodeUpdate(const flutter::SemanticsNodeUpdates update, float view_pixel_ratio)